From e26d7756eaa3148e445afc7386ea8c8214020199 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:59:02 -0700 Subject: [PATCH 01/36] test(sql): skip vectorSearch missing-plugin IT when k-NN is installed (#5492) VectorSearchIT.testExecutionWithoutKnnPluginReturnsCapabilityError asserts the "k-NN plugin not installed" capability error, which is only produced on a cluster without the k-NN plugin. The plugin's own JVM integ-test cluster has no k-NN, so the test passes there. The OpenSearch distribution bundles opensearch-knn, so on the distribution integ-test cluster the query reaches k-NN and fails with a different error ("not knn_vector type", HTTP 400), breaking the 3.x distribution integ-test run across all platforms. Guard the test with Assume.assumeFalse(isKnnPluginInstalled()) so it runs only on the no-k-NN cluster it was written for, mirroring the inverse assumeTrue guard already used by VectorSearchExecutionIT. No production code changes. Signed-off-by: ahkcs --- .../opensearch/sql/sql/VectorSearchIT.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index c10b3a219f6..8ae3167b40b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -8,8 +8,10 @@ import static org.hamcrest.Matchers.containsString; import java.io.IOException; +import org.junit.Assume; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; @@ -346,13 +348,22 @@ public void testEfficientModeRejectsScriptPredicate() throws IOException { } // ── k-NN plugin capability check ────────────────────────────────────── - // The default integ-test cluster does not have the k-NN plugin installed. Execution-path + // The plugin's own integ-test cluster does not have the k-NN plugin installed. Execution-path // queries against vectorSearch() should therefore fail with the clear "k-NN plugin missing" // error from KnnPluginCapability, while _explain continues to work because the capability // probe is deferred to scan open() and does not run during analysis/planning. @Test public void testExecutionWithoutKnnPluginReturnsCapabilityError() throws IOException { + // This test asserts the "k-NN plugin not installed" capability error, so it is only meaningful + // on a cluster WITHOUT the k-NN plugin. The OpenSearch distribution bundles opensearch-knn, so + // on the distribution integ-test cluster the query reaches k-NN and fails with a different + // ("not knn_vector type") error instead. Skip there; the missing-plugin path stays covered by + // the plugin's own JVM integ-test cluster, which does not ship k-NN. + Assume.assumeFalse( + "k-NN plugin is installed on this cluster; skipping missing-plugin capability check", + isKnnPluginInstalled()); + ResponseException ex = expectThrows( ResponseException.class, @@ -752,4 +763,16 @@ private void deleteAliasIfExists(String aliasName) { // Alias does not exist, which is fine. } } + + // Mirrors VectorSearchExecutionIT#isKnnPluginInstalled — opensearch-knn is bundled in the + // OpenSearch distribution but not in the plugin's own JVM integ-test cluster. + private static boolean isKnnPluginInstalled() { + try { + Response response = client().performRequest(new Request("GET", "/_cat/plugins?h=component")); + String body = new String(response.getEntity().getContent().readAllBytes()); + return body.contains("opensearch-knn"); + } catch (IOException e) { + return false; + } + } } From 5519bf0773b293ca40730cc28c3ad02f8ec3c7c3 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 1 Jun 2026 14:50:20 -0700 Subject: [PATCH 02/36] [Release 3.7.0] Patch release note 3.7 (#5494) Signed-off-by: Jialiang Liang --- release-notes/opensearch-sql.release-notes-3.7.0.0.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release-notes/opensearch-sql.release-notes-3.7.0.0.md b/release-notes/opensearch-sql.release-notes-3.7.0.0.md index d6898df38e6..7c486d296cf 100644 --- a/release-notes/opensearch-sql.release-notes-3.7.0.0.md +++ b/release-notes/opensearch-sql.release-notes-3.7.0.0.md @@ -32,6 +32,7 @@ Compatible with OpenSearch and OpenSearch Dashboards version 3.7.0 * Create parquet-backed test indices for `spath` command analytics-engine route ([#5441](https://github.com/opensearch-project/sql/pull/5441)) * Improve error messages for invalid index mapping by formatting index patterns and including underlying error details ([#5370](https://github.com/opensearch-project/sql/pull/5370)) * Initial implementation of report-builder interface for richer error context in responses ([#5266](https://github.com/opensearch-project/sql/pull/5266)) +* Validate materialized view subqueries against SQL grammar deny list ([#5485](https://github.com/opensearch-project/sql/pull/5485)) ### Bug Fixes @@ -65,6 +66,7 @@ Compatible with OpenSearch and OpenSearch Dashboards version 3.7.0 * Bump Gradle wrapper to 9.4.1 and workaround `@Ignore` test failure ([#5414](https://github.com/opensearch-project/sql/pull/5414)) * Fix link checker CI failure by excluding LinkedIn URLs ([#5461](https://github.com/opensearch-project/sql/pull/5461)) * Integration test cases for field-level security ([#5008](https://github.com/opensearch-project/sql/pull/5008)) +* Skip `vectorSearch()` missing-plugin integration test when the k-NN plugin is installed, fixing the distribution integ-test run since distributions bundle k-NN ([#5492](https://github.com/opensearch-project/sql/pull/5492)) ### Documentation From 050baee9e19521e3f864ad2e9a953ee7579d6af8 Mon Sep 17 00:00:00 2001 From: Finn Date: Mon, 1 Jun 2026 16:41:44 -0700 Subject: [PATCH 03/36] Add explain to AnalyticsExecutionEngine (#5442) Signed-off-by: Finn Carroll --- .../sql/executor/ExecutionEngine.java | 3 + .../analytics/AnalyticsExecutionEngine.java | 47 ++++ integ-test/build.gradle | 40 ++++ .../analytics/AnalyticsEngineProfileIT.java | 218 ++++++++++++++++++ .../plugin/rest/RestUnifiedQueryAction.java | 54 ++++- 5 files changed, 356 insertions(+), 6 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index da8eae41355..2a5d392a149 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -13,6 +13,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import org.apache.calcite.rel.RelNode; +import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.common.response.ResponseListener; @@ -80,6 +81,8 @@ class QueryResponse { private final Schema schema; private final List results; private final Cursor cursor; + @lombok.Setter private QueryProfile profile; + @lombok.Setter private Throwable error; } @Data diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java index 51854f16fb7..733c603a761 100644 --- a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java @@ -17,6 +17,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.opensearch.analytics.exec.QueryPlanExecutor; +import org.opensearch.analytics.exec.profile.ProfiledResult; import org.opensearch.analytics.schema.BinaryType; import org.opensearch.analytics.schema.IpType; import org.opensearch.common.network.InetAddresses; @@ -145,6 +146,52 @@ public void explain( } } + /** + * Executes the query with profiling enabled. Returns results + stage timing profile. Called when + * {@code profile=true} is set on the request. + */ + public void executeWithProfile( + RelNode plan, + CalcitePlanContext context, + org.opensearch.analytics.QueryRequestContext queryCtx, + ResponseListener listener) { + + planExecutor.executeWithProfile( + plan, + queryCtx, + new ActionListener<>() { + @Override + public void onResponse(ProfiledResult result) { + try { + // ProfiledResult delivers the profile on BOTH success and failure paths + // so users get stage timing visibility even when a query partially fails. + QueryResponse response = buildProfiledResponse(plan, result); + listener.onResponse(response); + } catch (Exception e) { + listener.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private QueryResponse buildProfiledResponse(RelNode plan, ProfiledResult result) { + List fields = plan.getRowType().getFieldList(); + Schema schema = buildSchema(fields); + List results = + result.rows() != null ? convertRows(result.rows(), fields) : List.of(); + QueryResponse response = new QueryResponse(schema, results, Cursor.None); + response.setProfile(result.profile()); + if (!result.isSuccess()) { + response.setError(result.failure()); + } + return response; + } + private List convertRows(Iterable rows, List fields) { List results = new ArrayList<>(); for (Object[] row : rows) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7d48e415204..51d3e502f0a 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -525,6 +525,43 @@ testClusters.analyticsEngineSecurityIT { configureSecurityPlugin(testClusters.analyticsEngineSecurityIT) +task analyticsEngineProfileIT(type: RestIntegTestTask) { + dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip, downloadArrowBaseZip, downloadAnalyticsBackendLuceneZip, downloadParquetDataFormatZip, downloadCompositeEngineZip, downloadAnalyticsBackendDatafusionZip + dependsOn ':opensearch-sql-plugin:bundlePlugin' + + systemProperty 'tests.security.manager', 'false' + + filter { + includeTestsMatching 'org.opensearch.sql.analytics.AnalyticsEngineProfileIT' + } +} + +testClusters.analyticsEngineProfileIT { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowBasePlugin()) + plugin(getArrowFlightRpcPlugin()) + plugin(getAnalyticsEnginePlugin()) + plugin(getAnalyticsBackendLucenePlugin()) + plugin(getAnalyticsBackendDatafusionPlugin()) + plugin(getParquetDataFormatPlugin()) + plugin(getCompositeEnginePlugin()) + plugin ":opensearch-sql-plugin" + // Arrow Flight / streaming transport requirements + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' + // Native library path for DataFusion/parquet — pass via -PnativeLibPath=/path/to/release/ + if (project.findProperty('nativeLibPath')) { + systemProperty 'java.library.path', project.findProperty('nativeLibPath') + } +} + task integJdbcTest(type: RestIntegTestTask) { testClusters.findAll {c -> c.clusterName == "integJdbcTest"}.first().with { plugin ":opensearch-sql-plugin" @@ -708,6 +745,9 @@ integTest { exclude 'org/opensearch/sql/doctest/**/*IT.class' exclude 'org/opensearch/sql/correctness/**' + // Exclude tests that require the analytics engine plugin stack (run separately via dedicated tasks) + exclude 'org/opensearch/sql/analytics/AnalyticsEngineProfileIT.class' + // Explain IT is dependent on internal implementation of old engine so it's not necessary // to run these with new engine and not necessary to make this consistent with old engine. exclude 'org/opensearch/sql/legacy/ExplainIT.class' diff --git a/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java b/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java new file mode 100644 index 00000000000..4a28115897a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java @@ -0,0 +1,218 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.analytics; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +/** + * Integration tests for the analytics engine profile API (profile=true). Verifies that PPL and SQL + * queries with profile=true return stage profiling, and that /_explain returns only the logical + * plan without execution. + * + *

Runs against the analyticsEngineProfileIT cluster which has the full analytics engine stack. + */ +public class AnalyticsEngineProfileIT extends OpenSearchRestTestCase { + + private static final String INDEX = "profile_test"; + private static boolean initialized = false; + + private void ensureSetup() throws IOException { + if (initialized) return; + enableCalcite(); + createCompositeIndex(); + ingestData(); + initialized = true; + } + + private void enableCalcite() throws IOException { + Request req = new Request("PUT", "/_cluster/settings"); + req.setJsonEntity("{\"persistent\":{\"plugins.calcite.enabled\":true}}"); + client().performRequest(req); + } + + private void createCompositeIndex() throws IOException { + try { + Request req = new Request("PUT", "/" + INDEX); + req.setJsonEntity( + """ + { + "settings": { + "number_of_shards": 2, + "number_of_replicas": 0, + "index.pluggable.dataformat.enabled": true, + "index.pluggable.dataformat": "composite" + }, + "mappings": { + "properties": { + "name": {"type": "keyword"}, + "score": {"type": "double"} + } + } + } + """); + client().performRequest(req); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 400) throw e; + // Index already exists + } + } + + private void ingestData() throws IOException { + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + bulk.setJsonEntity( + String.format( + Locale.ROOT, + """ + {"index":{"_index":"%s"}} + {"name":"alice","score":95.5} + {"index":{"_index":"%s"}} + {"name":"bob","score":87.3} + {"index":{"_index":"%s"}} + {"name":"carol","score":91.0} + """, + INDEX, + INDEX, + INDEX)); + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/x-ndjson"); + bulk.setOptions(opts); + client().performRequest(bulk); + } + + @Test + public void testPplProfileReturnsStages() throws IOException { + ensureSetup(); + JSONObject result = + executeWithProfile("source = " + INDEX + " | stats avg(score) by name", "/_plugins/_ppl"); + + assertTrue("has schema", result.has("schema")); + assertTrue("has datarows", result.has("datarows")); + assertTrue("has profile", result.has("profile")); + + JSONObject profile = result.getJSONObject("profile"); + assertTrue("has query_id", profile.has("query_id")); + assertTrue("has planning_time_ms", profile.has("planning_time_ms")); + assertTrue("has execution_time_ms", profile.has("execution_time_ms")); + assertTrue("has full_plan", profile.has("full_plan")); + + JSONArray stages = profile.getJSONArray("stages"); + assertTrue("at least one stage", stages.length() >= 1); + + JSONObject stage = stages.getJSONObject(0); + assertTrue("stage has stage_id", stage.has("stage_id")); + assertTrue("stage has execution_type", stage.has("execution_type")); + assertTrue("stage has state", stage.has("state")); + assertTrue("stage has elapsed_ms", stage.has("elapsed_ms")); + assertTrue("stage has tasks", stage.has("tasks")); + } + + @Test + public void testSqlProfileReturnsStages() throws IOException { + ensureSetup(); + JSONObject result = executeWithProfile("SELECT * FROM " + INDEX, "/_plugins/_sql"); + + assertTrue("has schema", result.has("schema")); + assertTrue("has datarows", result.has("datarows")); + assertTrue("has profile", result.has("profile")); + + JSONObject profile = result.getJSONObject("profile"); + assertTrue("has query_id", profile.has("query_id")); + assertTrue("has stages", profile.has("stages")); + JSONArray stages = profile.getJSONArray("stages"); + assertTrue("at least one stage", stages.length() >= 1); + } + + @Test + public void testPplProfileStagesShowSucceeded() throws IOException { + ensureSetup(); + JSONObject result = + executeWithProfile("source = " + INDEX + " | fields name, score", "/_plugins/_ppl"); + + JSONObject profile = result.getJSONObject("profile"); + JSONArray stages = profile.getJSONArray("stages"); + + for (int i = 0; i < stages.length(); i++) { + JSONObject stage = stages.getJSONObject(i); + assertEquals("stage succeeded", "SUCCEEDED", stage.getString("state")); + assertTrue("elapsed_ms non-negative", stage.getLong("elapsed_ms") >= 0); + } + } + + @Test + public void testPplProfileTasksHaveNodeAndTiming() throws IOException { + ensureSetup(); + JSONObject result = + executeWithProfile("source = " + INDEX + " | fields name", "/_plugins/_ppl"); + + JSONObject profile = result.getJSONObject("profile"); + JSONArray stages = profile.getJSONArray("stages"); + + boolean foundTasks = false; + for (int i = 0; i < stages.length(); i++) { + JSONArray tasks = stages.getJSONObject(i).getJSONArray("tasks"); + if (tasks.length() > 0) { + foundTasks = true; + JSONObject task = tasks.getJSONObject(0); + assertTrue("task has node", task.has("node")); + assertTrue("task has state", task.has("state")); + assertTrue("task has elapsed_ms", task.has("elapsed_ms")); + } + } + assertTrue("at least one stage has tasks", foundTasks); + } + + @Test + public void testPplExplainReturnsOnlyPlan() throws IOException { + ensureSetup(); + Request request = new Request("POST", "/_plugins/_ppl/_explain"); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"source = %s | fields name, score\"}", INDEX)); + Response response = client().performRequest(request); + JSONObject result = new JSONObject(entityAsString(response)); + + assertTrue("has calcite", result.has("calcite")); + JSONObject calcite = result.getJSONObject("calcite"); + assertTrue("has logical", calcite.has("logical")); + assertFalse("no profile in explain", calcite.has("profile")); + } + + @Test + public void testSqlExplainReturnsOnlyPlan() throws IOException { + ensureSetup(); + Request request = new Request("POST", "/_plugins/_sql/_explain"); + request.setJsonEntity(String.format(Locale.ROOT, "{\"query\": \"SELECT * FROM %s\"}", INDEX)); + Response response = client().performRequest(request); + JSONObject result = new JSONObject(entityAsString(response)); + + assertTrue("has calcite", result.has("calcite")); + JSONObject calcite = result.getJSONObject("calcite"); + assertTrue("has logical", calcite.has("logical")); + assertFalse("no profile in explain", calcite.has("profile")); + } + + private JSONObject executeWithProfile(String query, String endpoint) throws IOException { + Request request = new Request("POST", endpoint); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"profile\": true}", query)); + Response response = client().performRequest(request); + return new JSONObject(entityAsString(response)); + } + + private static String entityAsString(Response response) throws IOException { + return new String( + response.getEntity().getContent().readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 844513ed836..842dd2f1315 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -17,6 +17,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.ThreadContext; import org.opensearch.analytics.exec.QueryPlanExecutor; +import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; @@ -137,7 +138,10 @@ public void execute( // schema we plan against and the state the executor uses are the same view. org.opensearch.analytics.QueryRequestContext queryCtx = contextProvider.getContext(); - UnifiedQueryContext context = buildContext(queryType, profiling, queryCtx); + // Disable SQL-layer phase profiling when analytics engine profiling is active. + // Our QueryProfile (stages, tasks, timing) is strictly more detailed and replaces + // it. + UnifiedQueryContext context = buildContext(queryType, false, queryCtx); ActionListener closingListener = wrapWithContextClose(context, listener); try { @@ -145,11 +149,19 @@ public void execute( RelNode plan = planner.plan(query); CalcitePlanContext planContext = context.getPlanContext(); plan = addQuerySizeLimit(plan, planContext); - analyticsEngine.execute( - plan, - planContext, - queryCtx, - createQueryListener(queryType, closingListener)); + if (profiling) { + analyticsEngine.executeWithProfile( + plan, + planContext, + queryCtx, + createQueryListener(queryType, closingListener)); + } else { + analyticsEngine.execute( + plan, + planContext, + queryCtx, + createQueryListener(queryType, closingListener)); + } } catch (Exception e) { closingListener.onFailure(e); } @@ -270,6 +282,10 @@ public void onResponse(QueryResponse response) { formatter.format( new QueryResult( response.getSchema(), response.getResults(), response.getCursor(), langSpec)); + if (response.getProfile() != null) { + // Append profile and error (if any) to the JSON response + result = appendProfileToJson(result, response.getProfile(), response.getError()); + } transportListener.onResponse(new TransportPPLQueryResponse(result)); } @@ -280,6 +296,32 @@ public void onFailure(Exception e) { }; } + private static String appendProfileToJson(String json, QueryProfile profile, Throwable error) { + try { + StringBuilder extra = new StringBuilder(); + // Append profile + org.opensearch.core.xcontent.XContentBuilder builder = + org.opensearch.common.xcontent.XContentFactory.jsonBuilder(); + profile.toXContent(builder, org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS); + extra.append(",\"profile\":").append(builder.toString()); + // Append error if query partially failed + if (error != null) { + extra + .append(",\"error\":{\"type\":\"") + .append(error.getClass().getSimpleName()) + .append("\",\"reason\":\"") + .append(error.getMessage() != null ? error.getMessage().replace("\"", "\\\"") : "") + .append("\"}"); + } + if (json.endsWith("}")) { + return json.substring(0, json.length() - 1) + extra + "}"; + } + return json; + } catch (Exception e) { + return json; + } + } + private static Runnable withCurrentContext(final Runnable task) { final Map currentContext = ThreadContext.getImmutableContext(); return () -> { From 4c04a4dbebc2334d60badb4511ef057acd423861 Mon Sep 17 00:00:00 2001 From: Songkan Tang Date: Tue, 2 Jun 2026 14:15:51 +0800 Subject: [PATCH 04/36] Fix transpose VARCHAR length drift in unpivot/pivot lowering (#5479) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix transpose VARCHAR length drift in unpivot/pivot lowering PPL `transpose` lowers via `RelBuilder.unpivot()` + `pivot()`. The unpivot synthesizes a VALUES leaf carrying axis literals (the input field names), e.g. `VALUES('firstname'), ('age'), ('balance')`. Calcite types each RexLiteral as `CHAR(literalLen)` and types the VALUES column as `CHAR(maxAxisLiteralLen)` — the longest literal wins on column-level type inference. This bites the analytics-engine route end-to-end: 1. After unpivot the `column` axis column is `CHAR(9)` (from "firstname"). Through Calcite's TRIM (`TO_VARYING`) it becomes `VARCHAR(9)`. 2. The `_value_transpose_` value column is built from `CAST(input_field AS VARCHAR)` — unbounded VARCHAR. 3. `MAX(_value_transpose_)` aggCall is created with declared return type = unbounded VARCHAR (inferred from arg 0 at call-construction time). 4. The downstream non-prefix groupSet aggregate (`group=[{1}], MAX($0)`) splits into PARTIAL/FINAL on the analytics path. PARTIAL hoists group keys to the output prefix, so FINAL's `argList=[0]` reads the group-key slot — `VARCHAR(9)` — instead of the agg-state slot. Calcite's `Aggregate.` then runs `typeMatchesInferred` and rejects the plan: declared `VARCHAR` ≠ inferred `VARCHAR(9)`. 5. Even when the aggregate validation passes, the substrait/Arrow path sees `FixedChar(maxAxisLiteralLen)` schema vs runtime arrays whose actual values are shorter (e.g. "age" with length 3) and trips `Row field type (FixedChar{length=3}) does not match schema field type (FixedChar{length=9})`. Two fixes, both in the lowering site: * Build every axis literal at the same `CHAR(maxAxisLiteralLen)` type. Calcite then space-pads the shorter literals at value-construction time, so the runtime CHAR vector and the declared schema both have the same fixed length. The downstream TRIM strips the padding. * Wrap the trimmed-axis group key in an explicit `CAST(... AS VARCHAR)` to unbounded VARCHAR. This makes the group key type match `_value_transpose_`'s unbounded VARCHAR end-to-end, so the aggregate's row-type check sees consistent types regardless of which side the analytics-engine split rule places the group key on. These have to live in sql plugin, not in the analytics-engine planner: the typing decisions are made by Calcite's `RelBuilder.unpivot()` implementation when it constructs the VALUES leaf — long before any analytics-engine rule sees the plan. By the time the plan reaches the analytics-engine route, the precision drift is already baked into the RelDataType chain. Fixing it downstream would require pattern-matching on transpose-shaped sub-trees inside the planner, which is fragile and mis-attributes the root cause. The lowering author owns the type contract for the operators it emits. Adds: - `testTransposeColumnAxisUsesUnboundedVarchar` regression assertion pinning the output `column` field's type to unbounded VARCHAR. Catches any future change that re-introduces axis-literal precision into the group key. - Updated plan-shape assertions across the existing transpose tests to reflect the padded axis literals (`'cnt '`, `'COMM '`, etc.) and the `CAST(TRIM(...) AS VARCHAR)` group key. Verified end-to-end: `CalciteTransposeCommandIT` 5/5 pass with `tests.analytics.parquet_indices=true`. Signed-off-by: Songkan Tang * Update explain_transpose.yaml expected plan for new lowering shape CalciteExplainIT.testTransposeExplain regenerated against the updated transpose lowering: axis literals are now padded to a uniform CHAR(N) (N = max axis literal length, i.e. 14 for 'account_number'), and the group-key TRIM output is wrapped in a CAST(... AS VARCHAR) to unbounded VARCHAR. The plan-shape diff exactly mirrors the documented behavior change in the parent commit: * `$f20=[TRIM(...)]` → `$f20=[CAST(TRIM(...)):VARCHAR NOT NULL]` * axis literals e.g. 'firstname' → 'firstname ' (padded) * LogicalValues row type tuples are correspondingly padded Verified locally: `./gradlew :integ-test:integTestRemote --tests "*CalciteExplainIT.testTransposeExplain"` passes. Signed-off-by: Songkan Tang --------- Signed-off-by: Songkan Tang --- .../sql/calcite/CalciteRelNodeVisitor.java | 19 ++- .../calcite/explain_transpose.yaml | 13 +- .../ppl/calcite/CalcitePPLTransposeTest.java | 125 +++++++++--------- 3 files changed, 85 insertions(+), 72 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 335b48a45d9..c475c9e2b0b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -915,6 +915,9 @@ public RelNode visitTranspose( RelBuilder b = context.relBuilder; RexBuilder rx = context.rexBuilder; RelDataType varchar = rx.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + int axisLiteralLength = fieldNames.stream().mapToInt(String::length).max().orElse(0); + RelDataType axisLiteralType = + rx.getTypeFactory().createSqlType(SqlTypeName.CHAR, axisLiteralLength); // Step 1: ROW_NUMBER b.projectPlus( @@ -932,18 +935,22 @@ public RelNode visitTranspose( .map( f -> Map.entry( - ImmutableList.of(rx.makeLiteral(f)), + ImmutableList.of( + (RexLiteral) rx.makeLiteral(f, axisLiteralType, false, false)), ImmutableList.of((RexNode) rx.makeCast(varchar, b.field(f), true)))) .collect(Collectors.toList())); // Step 3: Trim spaces from columnName column before pivot RexNode trimmedColumnName = - context.rexBuilder.makeCall( - SqlStdOperatorTable.TRIM, - context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), - context.rexBuilder.makeLiteral(" "), - b.field(columnName)); + context.rexBuilder.makeCast( + varchar, + context.rexBuilder.makeCall( + SqlStdOperatorTable.TRIM, + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), + context.rexBuilder.makeLiteral(" "), + b.field(columnName)), + true); // Step 4: PIVOT b.pivot( diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml index d0a2f80d866..903980bed98 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml @@ -3,20 +3,21 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(column_names=[$0], row 1=[$1], row 2=[$2], row 3=[$3], row 4=[$4]) LogicalAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0) FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5]) - LogicalProject(_value_transpose_=[CAST($19):VARCHAR NOT NULL], $f20=[TRIM(FLAG(BOTH), ' ', $18)], $f21=[=($17, 1)], $f22=[=($17, 2)], $f23=[=($17, 3)], $f24=[=($17, 4)]) + LogicalProject(_value_transpose_=[CAST($19):VARCHAR NOT NULL], $f20=[CAST(TRIM(FLAG(BOTH), ' ', $18)):VARCHAR NOT NULL], $f21=[=($17, 1)], $f22=[=($17, 2)], $f23=[=($17, 3)], $f24=[=($17, 4)]) LogicalFilter(condition=[IS NOT NULL($19)]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_transpose_=[$17], column_names=[$18], _value_transpose_=[CASE(=($18, 'account_number'), CAST($0):VARCHAR NOT NULL, =($18, 'firstname'), CAST($1):VARCHAR NOT NULL, =($18, 'address'), CAST($2):VARCHAR NOT NULL, =($18, 'balance'), CAST($3):VARCHAR NOT NULL, =($18, 'gender'), CAST($4):VARCHAR NOT NULL, =($18, 'city'), CAST($5):VARCHAR NOT NULL, =($18, 'employer'), CAST($6):VARCHAR NOT NULL, =($18, 'state'), CAST($7):VARCHAR NOT NULL, =($18, 'age'), CAST($8):VARCHAR NOT NULL, =($18, 'email'), CAST($9):VARCHAR NOT NULL, =($18, 'lastname'), CAST($10):VARCHAR NOT NULL, null:NULL)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_transpose_=[$17], column_names=[$18], _value_transpose_=[CASE(=($18, 'account_number'), CAST($0):VARCHAR NOT NULL, =($18, 'firstname '), CAST($1):VARCHAR NOT NULL, =($18, 'address '), CAST($2):VARCHAR NOT NULL, =($18, 'balance '), CAST($3):VARCHAR NOT NULL, =($18, 'gender '), CAST($4):VARCHAR NOT NULL, =($18, 'city '), CAST($5):VARCHAR NOT NULL, =($18, 'employer '), CAST($6):VARCHAR NOT NULL, =($18, 'state '), CAST($7):VARCHAR NOT NULL, =($18, 'age '), CAST($8):VARCHAR NOT NULL, =($18, 'email '), CAST($9):VARCHAR NOT NULL, =($18, 'lastname '), CAST($10):VARCHAR NOT NULL, null:NULL)]) LogicalJoin(condition=[true], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_transpose_=[ROW_NUMBER() OVER ()]) LogicalSort(fetch=[5]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalValues(tuples=[[{ 'account_number' }, { 'firstname' }, { 'address' }, { 'balance' }, { 'gender' }, { 'city' }, { 'employer' }, { 'state' }, { 'age' }, { 'email' }, { 'lastname' }]]) + LogicalValues(tuples=[[{ 'account_number' }, { 'firstname ' }, { 'address ' }, { 'balance ' }, { 'gender ' }, { 'city ' }, { 'employer ' }, { 'state ' }, { 'age ' }, { 'email ' }, { 'lastname ' }]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0) FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=['account_number'], expr#14=[=($t12, $t13)], expr#15=[CAST($t0):VARCHAR NOT NULL], expr#16=['firstname'], expr#17=[=($t12, $t16)], expr#18=[CAST($t1):VARCHAR NOT NULL], expr#19=['address'], expr#20=[=($t12, $t19)], expr#21=[CAST($t2):VARCHAR NOT NULL], expr#22=['balance'], expr#23=[=($t12, $t22)], expr#24=[CAST($t3):VARCHAR NOT NULL], expr#25=['gender'], expr#26=[=($t12, $t25)], expr#27=[CAST($t4):VARCHAR NOT NULL], expr#28=['city'], expr#29=[=($t12, $t28)], expr#30=[CAST($t5):VARCHAR NOT NULL], expr#31=['employer'], expr#32=[=($t12, $t31)], expr#33=[CAST($t6):VARCHAR NOT NULL], expr#34=['state'], expr#35=[=($t12, $t34)], expr#36=[CAST($t7):VARCHAR NOT NULL], expr#37=['age'], expr#38=[=($t12, $t37)], expr#39=[CAST($t8):VARCHAR NOT NULL], expr#40=['email'], expr#41=[=($t12, $t40)], expr#42=[CAST($t9):VARCHAR NOT NULL], expr#43=['lastname'], expr#44=[=($t12, $t43)], expr#45=[CAST($t10):VARCHAR NOT NULL], expr#46=[null:NULL], expr#47=[CASE($t14, $t15, $t17, $t18, $t20, $t21, $t23, $t24, $t26, $t27, $t29, $t30, $t32, $t33, $t35, $t36, $t38, $t39, $t41, $t42, $t44, $t45, $t46)], expr#48=[CAST($t47):VARCHAR NOT NULL], expr#49=[FLAG(BOTH)], expr#50=[' '], expr#51=[TRIM($t49, $t50, $t12)], expr#52=[1], expr#53=[=($t11, $t52)], expr#54=[2], expr#55=[=($t11, $t54)], expr#56=[3], expr#57=[=($t11, $t56)], expr#58=[4], expr#59=[=($t11, $t58)], _value_transpose_=[$t48], $f20=[$t51], $f21=[$t53], $f22=[$t55], $f23=[$t57], $f24=[$t59]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=['account_number'], expr#14=[=($t12, $t13)], expr#15=[CAST($t0):VARCHAR NOT NULL], expr#16=['firstname '], expr#17=[=($t12, $t16)], expr#18=[CAST($t1):VARCHAR NOT NULL], expr#19=['address '], expr#20=[=($t12, $t19)], expr#21=[CAST($t2):VARCHAR NOT NULL], expr#22=['balance '], expr#23=[=($t12, $t22)], expr#24=[CAST($t3):VARCHAR NOT NULL], expr#25=['gender '], expr#26=[=($t12, $t25)], expr#27=[CAST($t4):VARCHAR NOT NULL], expr#28=['city '], expr#29=[=($t12, $t28)], expr#30=[CAST($t5):VARCHAR NOT NULL], expr#31=['employer '], expr#32=[=($t12, $t31)], expr#33=[CAST($t6):VARCHAR NOT NULL], expr#34=['state '], expr#35=[=($t12, $t34)], expr#36=[CAST($t7):VARCHAR NOT NULL], expr#37=['age '], expr#38=[=($t12, $t37)], expr#39=[CAST($t8):VARCHAR NOT NULL], expr#40=['email '], expr#41=[=($t12, $t40)], expr#42=[CAST($t9):VARCHAR NOT NULL], expr#43=['lastname '], expr#44=[=($t12, $t43)], expr#45=[CAST($t10):VARCHAR NOT NULL], expr#46=[null:NULL], expr#47=[CASE($t14, $t15, $t17, $t18, $t20, $t21, $t23, $t24, $t26, $t27, $t29, $t30, $t32, $t33, $t35, $t36, $t38, $t39, $t41, $t42, $t44, $t45, $t46)], expr#48=[CAST($t47):VARCHAR NOT NULL], expr#49=[FLAG(BOTH)], expr#50=[' '], expr#51=[TRIM($t49, $t50, $t12)], expr#52=[CAST($t51):VARCHAR NOT NULL], expr#53=[1], expr#54=[=($t11, $t53)], expr#55=[2], expr#56=[=($t11, $t55)], expr#57=[3], expr#58=[=($t11, $t57)], expr#59=[4], expr#60=[=($t11, $t59)], _value_transpose_=[$t48], $f20=[$t52], $f21=[$t54], $f22=[$t56], $f23=[$t58], $f24=[$t60]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[Sarg['account_number', 'address':CHAR(14), 'age':CHAR(14), 'balance':CHAR(14), 'city':CHAR(14), 'email':CHAR(14), 'employer':CHAR(14), 'firstname':CHAR(14), 'gender':CHAR(14), 'lastname':CHAR(14), 'state':CHAR(14)]:CHAR(14)], expr#2=[SEARCH($t0, $t1)], column_names=[$t0], $condition=[$t2]) - EnumerableValues(tuples=[[{ 'account_number' }, { 'firstname' }, { 'address' }, { 'balance' }, { 'gender' }, { 'city' }, { 'employer' }, { 'state' }, { 'age' }, { 'email' }, { 'lastname' }]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[Sarg['account_number', 'address ', 'age ', 'balance ', 'city ', 'email ', 'employer ', 'firstname ', 'gender ', 'lastname ', 'state ']:CHAR(14)], expr#2=[SEARCH($t0, $t1)], column_names=[$t0], $condition=[$t2]) + EnumerableValues(tuples=[[{ 'account_number' }, { 'firstname ' }, { 'address ' }, { 'balance ' }, { 'gender ' }, { 'city ' }, { 'employer ' }, { 'state ' }, { 'age ' }, { 'email ' }, { 'lastname ' }]]) + diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java index 69bc1ae2638..a9be30cb147 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java @@ -6,7 +6,10 @@ package org.opensearch.sql.ppl.calcite; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.test.CalciteAssert; +import org.junit.Assert; import org.junit.Test; public class CalcitePPLTransposeTest extends CalcitePPLAbstractTest { @@ -25,8 +28,8 @@ public void testSimpleCountWithTranspose() { + " FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5], row" + " 5_null=[MAX($0) FILTER $6])\n" + " LogicalProject(_value_transpose_=[CAST($3):VARCHAR NOT NULL]," - + " $f4=[TRIM(FLAG(BOTH), ' '," - + " $2)], $f5=[=($1, 1)], $f6=[=($1, 2)], $f7=[=($1, 3)], $f8=[=($1, 4)], $f9=[=($1," + + " $f4=[CAST(TRIM(FLAG(BOTH), ' ', $2)):VARCHAR NOT NULL], $f5=[=($1, 1)]," + + " $f6=[=($1, 2)], $f7=[=($1, 3)], $f8=[=($1, 4)], $f9=[=($1," + " 5)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" + " LogicalProject(c=[$0], _row_number_transpose_=[$1], column=[$2]," @@ -41,16 +44,13 @@ public void testSimpleCountWithTranspose() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column`) `column`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" - + " `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 2) `row 2`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" + "SELECT CAST(TRIM(`column`) AS STRING) `column`, MAX(CAST(`_value_transpose_` AS STRING))" + + " FILTER (WHERE `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_`" + + " AS STRING)) FILTER (WHERE `_row_number_transpose_` = 2) `row 2`," + + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE `_row_number_transpose_` =" + + " 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" + " `_row_number_transpose_` = 4) `row 4`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 5) `row 5`\n" + + " FILTER (WHERE `_row_number_transpose_` = 5) `row 5`\n" + "FROM (SELECT `t0`.`c`, `t0`.`_row_number_transpose_`, `t1`.`column`, CASE WHEN" + " `t1`.`column` = 'c' THEN CAST(`t0`.`c` AS STRING) ELSE NULL END" + " `_value_transpose_`\n" @@ -58,7 +58,7 @@ public void testSimpleCountWithTranspose() { + "FROM `scott`.`EMP`) `t0`\n" + "CROSS JOIN (VALUES ('c')) `t1` (`column`)) `t2`\n" + "WHERE `t2`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column`)"; + + "GROUP BY CAST(TRIM(`column`) AS STRING)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -75,14 +75,14 @@ public void testMultipleAggregatesWithAliasesTranspose() { + " FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5], row" + " 5_null=[MAX($0) FILTER $6])\n" + " LogicalProject(_value_transpose_=[CAST($6):VARCHAR NOT NULL]," - + " $f7=[TRIM(FLAG(BOTH), ' '," - + " $5)], $f8=[=($4, 1)], $f9=[=($4, 2)], $f10=[=($4, 3)], $f11=[=($4, 4)]," + + " $f7=[CAST(TRIM(FLAG(BOTH), ' ', $5)):VARCHAR NOT NULL], $f8=[=($4, 1)]," + + " $f9=[=($4, 2)], $f10=[=($4, 3)], $f11=[=($4, 4)]," + " $f12=[=($4, 5)])\n" + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + " LogicalProject(avg_sal=[$0], max_sal=[$1], min_sal=[$2], cnt=[$3]," + " _row_number_transpose_=[$4], column=[$5], _value_transpose_=[CASE(=($5, 'avg_sal')," + " NUMBER_TO_STRING($0), =($5, 'max_sal'), NUMBER_TO_STRING($1), =($5, 'min_sal')," - + " NUMBER_TO_STRING($2), =($5, 'cnt'), CAST($3):VARCHAR NOT NULL, null:NULL)])\n" + + " NUMBER_TO_STRING($2), =($5, 'cnt '), CAST($3):VARCHAR NOT NULL, null:NULL)])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalProject(avg_sal=[$0], max_sal=[$1], min_sal=[$2], cnt=[$3]," + " _row_number_transpose_=[ROW_NUMBER() OVER ()])\n" @@ -91,7 +91,7 @@ public void testMultipleAggregatesWithAliasesTranspose() { + " LogicalProject(SAL=[$5])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalValues(tuples=[[{ 'avg_sal' }, { 'max_sal' }, { 'min_sal' }, {" - + " 'cnt' }]])\n"; + + " 'cnt ' }]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "column=avg_sal; row 1=2073.214285; row 2=null; row 3=null; row 4=null; row 5=null\n" @@ -102,31 +102,28 @@ public void testMultipleAggregatesWithAliasesTranspose() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column`) `column`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" - + " `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 2) `row 2`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" + "SELECT CAST(TRIM(`column`) AS STRING) `column`, MAX(CAST(`_value_transpose_` AS STRING))" + + " FILTER (WHERE `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_`" + + " AS STRING)) FILTER (WHERE `_row_number_transpose_` = 2) `row 2`," + + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE `_row_number_transpose_` =" + + " 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" + " `_row_number_transpose_` = 4) `row 4`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 5) `row 5`\n" + + " FILTER (WHERE `_row_number_transpose_` = 5) `row 5`\n" + "FROM (SELECT `t1`.`avg_sal`, `t1`.`max_sal`, `t1`.`min_sal`, `t1`.`cnt`," + " `t1`.`_row_number_transpose_`, `t2`.`column`, CASE WHEN `t2`.`column` = 'avg_sal'" + " THEN NUMBER_TO_STRING(`t1`.`avg_sal`) WHEN `t2`.`column` = 'max_sal' THEN" + " NUMBER_TO_STRING(`t1`.`max_sal`) WHEN `t2`.`column` = 'min_sal' THEN" - + " NUMBER_TO_STRING(`t1`.`min_sal`) WHEN `t2`.`column` = 'cnt' THEN CAST(`t1`.`cnt` AS" - + " STRING) ELSE NULL END `_value_transpose_`\n" + + " NUMBER_TO_STRING(`t1`.`min_sal`) WHEN `t2`.`column` = 'cnt ' THEN" + + " CAST(`t1`.`cnt` AS STRING) ELSE NULL END `_value_transpose_`\n" + "FROM (SELECT AVG(`SAL`) `avg_sal`, MAX(`SAL`) `max_sal`, MIN(`SAL`) `min_sal`," + " COUNT(*) `cnt`, ROW_NUMBER() OVER () `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t1`\n" + "CROSS JOIN (VALUES ('avg_sal'),\n" + "('max_sal'),\n" + "('min_sal'),\n" - + "('cnt')) `t2` (`column`)) `t3`\n" + + "('cnt ')) `t2` (`column`)) `t3`\n" + "WHERE `t3`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column`)"; + + "GROUP BY CAST(TRIM(`column`) AS STRING)"; /* "SELECT `column`, MAX(CASE WHEN `__row_id__` = 1 THEN CAST(`value` AS STRING) ELSE NULL" @@ -164,18 +161,18 @@ public void testTransposeWithLimit() { + " LogicalAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0)" + " FILTER $3], row 3_null=[MAX($0) FILTER $4])\n" + " LogicalProject(_value_transpose_=[CAST($6):VARCHAR NOT NULL]," - + " $f7=[TRIM(FLAG(BOTH), ' '," - + " $5)], $f8=[=($4, 1)], $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + + " $f7=[CAST(TRIM(FLAG(BOTH), ' ', $5)):VARCHAR NOT NULL], $f8=[=($4, 1)]," + + " $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + " LogicalProject(ENAME=[$0], COMM=[$1], JOB=[$2], SAL=[$3]," + " _row_number_transpose_=[$4], column=[$5], _value_transpose_=[CASE(=($5, 'ENAME')," - + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM'), NUMBER_TO_STRING($1), =($5, 'JOB')," - + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL'), NUMBER_TO_STRING($3), null:NULL)])\n" + + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM '), NUMBER_TO_STRING($1), =($5, 'JOB ')," + + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL '), NUMBER_TO_STRING($3), null:NULL)])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalProject(ENAME=[$1], COMM=[$6], JOB=[$2], SAL=[$5]," + " _row_number_transpose_=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM' }, { 'JOB' }, { 'SAL'" + + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM ' }, { 'JOB ' }, { 'SAL '" + " }]])\n"; verifyLogical(root, expectedLogical); @@ -188,31 +185,39 @@ public void testTransposeWithLimit() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column`) `column`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" - + " `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 2) `row 2`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 3) `row 3`\n" + "SELECT CAST(TRIM(`column`) AS STRING) `column`, MAX(CAST(`_value_transpose_` AS STRING))" + + " FILTER (WHERE `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_`" + + " AS STRING)) FILTER (WHERE `_row_number_transpose_` = 2) `row 2`," + + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE `_row_number_transpose_` =" + + " 3) `row 3`\n" + "FROM (SELECT `t`.`ENAME`, `t`.`COMM`, `t`.`JOB`, `t`.`SAL`," + " `t`.`_row_number_transpose_`, `t0`.`column`, CASE WHEN `t0`.`column` = 'ENAME' THEN" - + " CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column` = 'COMM' THEN" - + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column` = 'JOB' THEN CAST(`t`.`JOB` AS" - + " STRING) WHEN `t0`.`column` = 'SAL' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE NULL END" + + " CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column` = 'COMM ' THEN" + + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column` = 'JOB ' THEN CAST(`t`.`JOB` AS" + + " STRING) WHEN `t0`.`column` = 'SAL ' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE NULL END" + " `_value_transpose_`\n" + "FROM (SELECT `ENAME`, `COMM`, `JOB`, `SAL`, ROW_NUMBER() OVER ()" + " `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t`\n" + "CROSS JOIN (VALUES ('ENAME'),\n" - + "('COMM'),\n" - + "('JOB'),\n" - + "('SAL')) `t0` (`column`)) `t1`\n" + + "('COMM '),\n" + + "('JOB '),\n" + + "('SAL ')) `t0` (`column`)) `t1`\n" + "WHERE `t1`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column`)"; + + "GROUP BY CAST(TRIM(`column`) AS STRING)"; verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testTransposeColumnAxisUsesUnboundedVarchar() { + RelNode root = getRelNode("source=EMP | fields ENAME, COMM, JOB, SAL | transpose 3"); + RelDataType columnType = root.getRowType().getFieldList().get(0).getType(); + + Assert.assertEquals(SqlTypeName.VARCHAR, columnType.getSqlTypeName()); + Assert.assertEquals(RelDataType.PRECISION_NOT_SPECIFIED, columnType.getPrecision()); + } + @Test public void testTransposeWithValueFieldNameCollision() { // Reproduce issue #5172: hardcoded 'value' unpivot column collides with @@ -237,19 +242,19 @@ public void testTransposeWithLimitColumnName() { + " LogicalAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0)" + " FILTER $3], row 3_null=[MAX($0) FILTER $4])\n" + " LogicalProject(_value_transpose_=[CAST($6):VARCHAR NOT NULL]," - + " $f7=[TRIM(FLAG(BOTH), ' '," - + " $5)], $f8=[=($4, 1)], $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + + " $f7=[CAST(TRIM(FLAG(BOTH), ' ', $5)):VARCHAR NOT NULL], $f8=[=($4, 1)]," + + " $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + " LogicalProject(ENAME=[$0], COMM=[$1], JOB=[$2], SAL=[$3]," + " _row_number_transpose_=[$4], column_names=[$5]," + " _value_transpose_=[CASE(=($5, 'ENAME')," - + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM'), NUMBER_TO_STRING($1), =($5, 'JOB')," - + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL'), NUMBER_TO_STRING($3), null:NULL)])\n" + + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM '), NUMBER_TO_STRING($1), =($5, 'JOB ')," + + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL '), NUMBER_TO_STRING($3), null:NULL)])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalProject(ENAME=[$1], COMM=[$6], JOB=[$2], SAL=[$5]," + " _row_number_transpose_=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM' }, { 'JOB' }, { 'SAL'" + + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM ' }, { 'JOB ' }, { 'SAL '" + " }]])\n"; verifyLogical(root, expectedLogical); @@ -261,7 +266,7 @@ public void testTransposeWithLimitColumnName() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column_names`) `column_names`," + "SELECT CAST(TRIM(`column_names`) AS STRING) `column_names`," + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" + " `_row_number_transpose_` = 1) `row 1`," + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" @@ -270,19 +275,19 @@ public void testTransposeWithLimitColumnName() { + " `_row_number_transpose_` = 3) `row 3`\n" + "FROM (SELECT `t`.`ENAME`, `t`.`COMM`, `t`.`JOB`, `t`.`SAL`," + " `t`.`_row_number_transpose_`, `t0`.`column_names`, CASE WHEN `t0`.`column_names` =" - + " 'ENAME' THEN CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column_names` = 'COMM' THEN" - + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column_names` = 'JOB' THEN CAST(`t`.`JOB`" - + " AS STRING) WHEN `t0`.`column_names` = 'SAL' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE" + + " 'ENAME' THEN CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column_names` = 'COMM ' THEN" + + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column_names` = 'JOB ' THEN CAST(`t`.`JOB`" + + " AS STRING) WHEN `t0`.`column_names` = 'SAL ' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE" + " NULL END `_value_transpose_`\n" + "FROM (SELECT `ENAME`, `COMM`, `JOB`, `SAL`, ROW_NUMBER() OVER ()" + " `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t`\n" + "CROSS JOIN (VALUES ('ENAME'),\n" - + "('COMM'),\n" - + "('JOB'),\n" - + "('SAL')) `t0` (`column_names`)) `t1`\n" + + "('COMM '),\n" + + "('JOB '),\n" + + "('SAL ')) `t0` (`column_names`)) `t1`\n" + "WHERE `t1`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column_names`)"; + + "GROUP BY CAST(TRIM(`column_names`) AS STRING)"; verifyPPLToSparkSQL(root, expectedSparkSql); } From 797e4fb1917f5b4a46df2485c172040ad867f558 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:58:34 -0700 Subject: [PATCH 05/36] Exclude AnalyticsEngineCompatIT from the main integTest task (#5501) * test(integ-test): exclude AnalyticsEngineCompatIT from the main integTest task AnalyticsEngineCompatIT (package org.opensearch.sql.plugin) is a smoke test that only makes sense against a cluster bundling the analytics-engine plugin stack. It is provisioned and run by the dedicated :analyticsEngineCompatIT task (testClusters.analyticsEngineCompat), which adds arrow-base, arrow-flight-rpc and analytics-engine. The main integTest task selects tests by exclusion and excluded org/opensearch/sql/security/** ("executed in another task") but never excluded the plugin package, so the smoke test also ran against the plain integTest/remoteCluster (neither of which has analytics-engine). There it is useless and exposed to suite-wide infra flakiness; in a recent CI run it was the descriptor recorded as failed after the test JVM wedged for ~4.5h. Mirror the existing security/** exclusion for org/opensearch/sql/plugin/**. Signed-off-by: Kai Huang * test(integ-test): narrow exclusion to the AnalyticsEngineCompatIT class Address review feedback: exclude the specific test class rather than the whole org/opensearch/sql/plugin/** package, matching the per-class exclude style already used in this task (e.g. legacy/ExplainIT.class). This keeps the exclusion intentional and avoids silently dropping any future test added to that package. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- integ-test/build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 51d3e502f0a..21741bffff3 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -766,6 +766,12 @@ integTest { // Exclude this IT, because they executed in another task (:integTestWithSecurity) exclude 'org/opensearch/sql/security/**' + // Exclude the analytics-engine smoke test, because it executes in another task + // (:analyticsEngineCompatIT) against a cluster that bundles the analytics-engine plugin + // stack. Running it here against the plain integTest/remoteCluster (no analytics-engine) + // is pointless and leaves it exposed to suite-wide infra flakiness. + exclude 'org/opensearch/sql/plugin/AnalyticsEngineCompatIT.class' + // Workaround for Gradle 9.4.1 ClassCastException in TestEventReporterAsListener.started // (line 58) — the bridge casts a parent test descriptor's reporter to // GroupTestEventReporterInternal but a class-level @Ignore produces a non-composite parent From 132f3b177dbd6c881b87ea0266b5d2a1547a38b3 Mon Sep 17 00:00:00 2001 From: Finn Date: Tue, 2 Jun 2026 16:46:40 -0700 Subject: [PATCH 06/36] Expand FGAC integration tests: exact permission, aliases, wildcards, index patterns (#5503) --- .../security/AnalyticsEngineSecurityIT.java | 170 ++++++++++++++++-- 1 file changed, 158 insertions(+), 12 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java index a9e36e9b470..626a7658dab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java @@ -23,6 +23,8 @@ public class AnalyticsEngineSecurityIT extends SecurityTestBase { private static final String TEST_INDEX = "analytics_security_test"; private static final String FORBIDDEN_INDEX = "analytics_forbidden_test"; + private static final String TEST_INDEX_2 = "analytics_security_extra"; + private static final String TEST_ALIAS = "analytics_alias"; private static final String ALLOWED_USER = "analytics_allowed_user"; private static final String ALLOWED_ROLE = "analytics_allowed_role"; @@ -32,6 +34,10 @@ public class AnalyticsEngineSecurityIT extends SecurityTestBase { private static final String SEARCH_ONLY_ROLE = "analytics_search_only_role"; private static final String WILDCARD_USER = "analytics_wildcard_user"; private static final String WILDCARD_ROLE = "analytics_wildcard_role"; + private static final String ALIAS_USER = "analytics_alias_user"; + private static final String ALIAS_ROLE = "analytics_alias_role"; + private static final String EXACT_PERM_USER = "analytics_exact_perm_user"; + private static final String EXACT_PERM_ROLE = "analytics_exact_perm_role"; private static boolean initialized = false; @@ -73,30 +79,39 @@ private void createTestIndices() throws IOException { // Create composite (analytics-engine-backed) indices so the SQL plugin routes // queries through the analytics engine's DefaultPlanExecutor. createCompositeIndex(TEST_INDEX); + createCompositeIndex(TEST_INDEX_2); + createCompositeIndex(FORBIDDEN_INDEX); + + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/x-ndjson"); + Request bulk = new Request("POST", "/_bulk"); bulk.addParameter("refresh", "true"); bulk.setJsonEntity( String.format( Locale.ROOT, "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"alice\", \"age\": 30}\n" - + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"bob\", \"age\": 25}\n", + + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"bob\", \"age\": 25}\n" + + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"carol\", \"age\": 28}\n" + + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"secret\", \"age\": 99}\n", TEST_INDEX, - TEST_INDEX)); - RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); - opts.addHeader("Content-Type", "application/x-ndjson"); + TEST_INDEX, + TEST_INDEX_2, + FORBIDDEN_INDEX)); bulk.setOptions(opts); client().performRequest(bulk); - createCompositeIndex(FORBIDDEN_INDEX); - Request bulkF = new Request("POST", "/_bulk"); - bulkF.addParameter("refresh", "true"); - bulkF.setJsonEntity( + // Create alias pointing to TEST_INDEX + Request aliasReq = new Request("POST", "/_aliases"); + aliasReq.setJsonEntity( String.format( Locale.ROOT, - "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"secret\", \"age\": 99}\n", - FORBIDDEN_INDEX)); - bulkF.setOptions(opts); - client().performRequest(bulkF); + """ + {"actions": [{"add": {"index": "%s", "alias": "%s"}}]} + """, + TEST_INDEX, + TEST_ALIAS)); + client().performRequest(aliasReq); } private void createCompositeIndex(String index) throws IOException { @@ -166,6 +181,30 @@ private void createSecurityRolesAndUsers() throws IOException { "indices:data/read*", "indices:admin/mappings/get", "indices:monitor/settings/get" }); createUser(WILDCARD_USER, WILDCARD_ROLE); + + // Role with access only to the alias — verifies security plugin resolves alias to + // concrete index and permits access when role's index_patterns matches the alias name. + createRoleWithPermissions( + ALIAS_ROLE, + TEST_ALIAS, + new String[] {"cluster:admin/opensearch/ppl", "cluster:admin/opensearch/sql"}, + new String[] { + "indices:data/read*", "indices:admin/mappings/get", "indices:monitor/settings/get" + }); + createUser(ALIAS_USER, ALIAS_ROLE); + + // Role with exactly indices:data/read/analytics/query — proves this specific permission + // is both necessary and sufficient for analytics engine queries. + createRoleWithPermissions( + EXACT_PERM_ROLE, + TEST_INDEX, + new String[] {"cluster:admin/opensearch/ppl", "cluster:admin/opensearch/sql"}, + new String[] { + "indices:data/read/analytics/query", + "indices:admin/mappings/get", + "indices:monitor/settings/get" + }); + createUser(EXACT_PERM_USER, EXACT_PERM_ROLE); } @Test @@ -216,6 +255,26 @@ public void testPPLQueryDeniedWithSearchPermissionOnly() throws IOException { executePPLAsUser( "source = " + TEST_INDEX + " | fields name, age", SEARCH_ONLY_USER)); assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + String body = org.opensearch.sql.legacy.TestUtils.getResponseBody(e.getResponse(), true); + assertTrue( + "Expected response to reference the missing analytics/query action, got: " + body, + body.contains("indices:data/read/analytics/query")); + } + + @Test + public void testPPLQueryAllowedWithExactAnalyticsQueryPermission() throws IOException { + // User has exactly indices:data/read/analytics/query (not a broad wildcard). + // Proves this specific permission is sufficient for analytics engine queries. + try { + JSONObject result = + executePPLAsUser("source = " + TEST_INDEX + " | fields name, age", EXACT_PERM_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for user with exact analytics/query permission", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } } @Test @@ -247,6 +306,93 @@ public void testPPLQueryDeniedWithWildcardPermissionOnNonMatchingIndex() throws assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); } + // --- Alias-based access tests --- + + @Test + public void testPPLQueryAllowedViaAlias() throws IOException { + // User's role has index_patterns: ["analytics_alias"]. Security plugin resolves the + // alias to the concrete index. Since AnalyticsQueryRequest uses strictExpandOpen(), + // IndexNameExpressionResolver resolves the alias and security matches it against the + // role's index_patterns which includes the alias name. + try { + JSONObject result = + executePPLAsUser("source = " + TEST_ALIAS + " | fields name, age", ALIAS_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for alias-permitted user", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + @Test + public void testPPLQueryDeniedViaAliasForUnauthorizedUser() throws IOException { + // DENIED_USER has no access to analytics_alias or the underlying index. + ResponseException e = + assertThrows( + ResponseException.class, + () -> executePPLAsUser("source = " + TEST_ALIAS + " | fields name, age", DENIED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLQueryAllowedViaConcreteIndexForAliasUser() throws IOException { + // ALIAS_USER's role has index_patterns: ["analytics_alias"]. In OpenSearch's security + // model, granting access to an alias also implicitly grants access to the underlying + // concrete index. This verifies the query succeeds via the concrete name. + try { + JSONObject result = + executePPLAsUser("source = " + TEST_INDEX + " | fields name, age", ALIAS_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for alias user querying concrete index", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + // --- Wildcard index pattern in query tests --- + + @Test + public void testPPLQueryWithWildcardIndexAllowed() throws IOException { + // WILDCARD_USER has index_patterns: ["analytics_security*"]. Query uses wildcard + // "analytics_security*" which resolves to analytics_security_test and + // analytics_security_extra — both match the role's pattern. + try { + JSONObject result = + executePPLAsUser("source = analytics_security* | fields name, age", WILDCARD_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for wildcard query with matching permissions", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + @Test + public void testPPLQueryWithWildcardIndexDenied() throws IOException { + // DENIED_USER has no access to any analytics_* indices. + ResponseException e = + assertThrows( + ResponseException.class, + () -> executePPLAsUser("source = analytics_security* | fields name, age", DENIED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLQueryWithWildcardIndexPartialAccessDenied() throws IOException { + // ALIAS_USER only has access to "analytics_alias" — not to "analytics_security*". + // A wildcard query expanding to indices the user lacks permission for should be denied. + ResponseException e = + assertThrows( + ResponseException.class, + () -> executePPLAsUser("source = analytics_security* | fields name, age", ALIAS_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + @Test public void testSQLQueryAllowedForAuthorizedUser() throws IOException { try { From ea39ffd9d377abfaeed19b15cb3dd94854292a54 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 3 Jun 2026 13:39:55 -0700 Subject: [PATCH 07/36] Add UNION ALL support to V3 SQL engine (#5506) Add UNION ALL set operations to the Calcite-based unified query path. The legacy V2 SQL engine rejects UNION with a SyntaxCheckException so queries fall back to the legacy engine appropriately. Signed-off-by: Chen Dai --- .../sql/api/parser/SqlV2QueryParser.java | 11 ++++++ .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 36 +++++++++++++++++++ .../org/opensearch/sql/ast/dsl/AstDSL.java | 5 +++ .../org/opensearch/sql/ast/tree/Union.java | 12 ++++++- .../sql/calcite/CalciteRelNodeVisitor.java | 6 ++-- sql/src/main/antlr/OpenSearchSQLParser.g4 | 3 +- .../opensearch/sql/sql/parser/AstBuilder.java | 6 ++++ .../sql/sql/parser/AstBuilderTest.java | 7 ++++ 8 files changed, 82 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java index d6280b829c7..d60ef7ed4ec 100644 --- a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java +++ b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java @@ -8,8 +8,11 @@ import static org.opensearch.sql.ast.dsl.AstDSL.existsSubquery; import static org.opensearch.sql.ast.dsl.AstDSL.inSubquery; import static org.opensearch.sql.ast.dsl.AstDSL.join; +import static org.opensearch.sql.ast.dsl.AstDSL.union; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -22,6 +25,7 @@ import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ExistsSubqueryExpressionAtomContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.InSubqueryPredicateContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.JoinClauseContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.UnionSelectContext; import org.opensearch.sql.sql.parser.AstBuilder; import org.opensearch.sql.sql.parser.AstExpressionBuilder; import org.opensearch.sql.sql.parser.AstStatementBuilder; @@ -81,6 +85,13 @@ private JoinType toJoinType(JoinClauseContext ctx) { }; } + @Override + public UnresolvedPlan visitUnionSelect(UnionSelectContext ctx) { + List datasets = + ctx.querySpecification().stream().map(this::visit).collect(Collectors.toList()); + return union(datasets); + } + /** * Expression builder with IN/EXISTS subquery support. Accesses the enclosing AstBuilder to * visit subquery plan nodes. Must be created via {@link #createExpressionBuilder()} because the diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index b818795609f..b435ea2da1a 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -200,6 +200,42 @@ WHERE age NOT IN (SELECT age FROM catalog.departments WHERE dept_name = 'Enginee """); } + @Test + public void testUnionAll() { + givenQuery( + """ + SELECT name FROM catalog.employees UNION ALL SELECT dept_name FROM catalog.departments + """) + .assertPlan( + """ + LogicalUnion(all=[true]) + LogicalProject(name=[$1]) + LogicalTableScan(table=[[catalog, employees]]) + LogicalProject(dept_name=[$1]) + LogicalTableScan(table=[[catalog, departments]]) + """); + } + + @Test + public void testMultiWayUnion() { + givenQuery( + """ + SELECT name FROM catalog.employees + UNION ALL SELECT dept_name FROM catalog.departments + UNION ALL SELECT name FROM catalog.employees + """) + .assertPlan( + """ + LogicalUnion(all=[true]) + LogicalProject(name=[$1]) + LogicalTableScan(table=[[catalog, employees]]) + LogicalProject(dept_name=[$1]) + LogicalTableScan(table=[[catalog, departments]]) + LogicalProject(name=[$1]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + @Test public void testNotExistsSubquery() { givenQuery( diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index f00a87ab40f..3a96137f42d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -84,6 +84,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -781,4 +782,8 @@ public static InSubquery inSubquery(UnresolvedPlan query, UnresolvedExpression.. public static ExistsSubquery existsSubquery(UnresolvedPlan query) { return new ExistsSubquery(query); } + + public static Union union(List datasets) { + return new Union(datasets); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Union.java b/core/src/main/java/org/opensearch/sql/ast/tree/Union.java index a96831567cb..7eefe89ce53 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Union.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Union.java @@ -21,15 +21,25 @@ @RequiredArgsConstructor @AllArgsConstructor public class Union extends UnresolvedPlan { + /** Input subplans (operands) combined by this UNION ALL. */ private final List datasets; + /** Whether inputs are unified to a common schema by name (PPL) vs combined positionally (SQL). */ + private boolean unifySchema; + + /** Optional cap on output rows (PPL {@code maxout}); {@code null} if unbounded. */ private Integer maxout; + /** PPL constructor: UNION ALL with schema unification. */ + public Union(List datasets, Integer maxout) { + this(datasets, true, maxout); + } + @Override public UnresolvedPlan attach(UnresolvedPlan child) { List newDatasets = ImmutableList.builder().add(child).addAll(datasets).build(); - return new Union(newDatasets, maxout); + return new Union(newDatasets, unifySchema, maxout); } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index c475c9e2b0b..cf7555f1ea7 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -3031,8 +3031,10 @@ public RelNode visitUnion(Union node, CalcitePlanContext context) { "Union command requires at least two datasets. Provided: " + inputNodes.size()); } - List unifiedInputs = - SchemaUnifier.buildUnifiedSchemaWithTypeCoercion(inputNodes, context); + List unifiedInputs = inputNodes; + if (node.isUnifySchema()) { + unifiedInputs = SchemaUnifier.buildUnifiedSchemaWithTypeCoercion(inputNodes, context); + } for (RelNode input : unifiedInputs) { context.relBuilder.push(input); diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 492f6dee9c6..5b52b9d3387 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -68,7 +68,8 @@ dmlStatement // Primary DML Statements selectStatement - : querySpecification # simpleSelect + : querySpecification # simpleSelect + | querySpecification (UNION ALL querySpecification)+ # unionSelect ; adminStatement diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index ee532a10ed9..aaed2ba5ec2 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -263,6 +263,12 @@ public UnresolvedPlan visitJoinClause(OpenSearchSQLParser.JoinClauseContext ctx) "JOIN is not supported in the V2 SQL engine. Falling back to legacy engine."); } + @Override + public UnresolvedPlan visitUnionSelect(OpenSearchSQLParser.UnionSelectContext ctx) { + throw new SyntaxCheckException( + "UNION is not supported in the V2 SQL engine. Falling back to legacy engine."); + } + @Override public UnresolvedPlan visitHavingClause(HavingClauseContext ctx) { AstHavingFilterBuilder builder = new AstHavingFilterBuilder(context.peek()); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 7869ba5cdad..9d504a32bb0 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -757,6 +757,13 @@ public UnresolvedPlan visitJoinClause(OpenSearchSQLParser.JoinClauseContext ctx) assertNotNull(new SQLSyntaxParser().parse(query).accept(builder)); } + @Test + public void union_throws_syntax_check_exception() { + assertThrows( + SyntaxCheckException.class, + () -> buildAST("SELECT name FROM t1 UNION ALL SELECT name FROM t2")); + } + @Test public void in_subquery_throws_syntax_check_exception() { assertThrows( From 6cff341e8282ec455c3c8d44fac4a8afed4df8e7 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Thu, 4 Jun 2026 11:29:16 -0700 Subject: [PATCH 08/36] [BugFix] Preserve SQL aggregate aliases in plan (#5509) SELECT col, COUNT(*) AS cnt only renames a column without changing the set or order, so RelBuilder.project (force=false) skips emitting the LogicalProject and the alias is dropped. Force the projection when an AS renames a field; the PPL path is unaffected since it never produces an AS RexCall through visitProject. Signed-off-by: Chen Dai --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 51 +++++++++++++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 21 +++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index b435ea2da1a..fb38f184925 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -277,6 +277,57 @@ public void selectExpressionWithoutFrom() { """); } + @Test + public void testGroupByAggregateAlias() { + givenQuery( + """ + SELECT department, SUM(age) AS total FROM catalog.employees GROUP BY department + """) + .assertPlan( + """ + LogicalProject(department=[$0], total=[$1]) + LogicalAggregate(group=[{0}], SUM(age)=[SUM($1)]) + LogicalProject(department=[$3], age=[$2]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testOrderByAggregateAlias() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt FROM catalog.employees + GROUP BY department ORDER BY cnt DESC LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(sort0=[$1], dir0=[DESC-nulls-last]) + LogicalProject(department=[$1], cnt=[$0]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[3]) + LogicalProject(COUNT(*)=[$1], department=[$0]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testAliasPreservedInOutputSchema() { + givenQuery("SELECT COUNT(*) AS cnt FROM catalog.employees").assertFields("cnt"); + + givenQuery("SELECT department, COUNT(*) AS cnt FROM catalog.employees GROUP BY department") + .assertFields("department", "cnt"); + + givenQuery("SELECT department, COUNT(*) FROM catalog.employees GROUP BY department") + .assertFields("department", "COUNT(*)"); + + givenQuery("SELECT MAX(age) + MIN(age) AS range_sum FROM catalog.employees") + .assertFields("range_sum"); + + givenQuery("SELECT id, name, age AS years, department FROM catalog.employees") + .assertFields("id", "name", "years", "department"); + } + @Test public void testHavingMaxCol() { givenQuery( diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index cf7555f1ea7..91a30361a20 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -481,11 +481,30 @@ public RelNode visitProject(Project node, CalcitePlanContext context) { if (!context.isResolvingSubquery()) { context.setProjectVisited(true); } - context.relBuilder.project(expandedFields); + + // Force the projection on a rename: without it, Calcite omits the project node when the + // columns are unchanged (same fields and order), so an alias like COUNT(*) AS cnt is lost. + boolean force = isRenameFieldsProject(expandedFields, currentFields); + context.relBuilder.project(expandedFields, ImmutableList.of(), force); } return context.relBuilder.peek(); } + private static boolean isRenameFieldsProject(List fields, List currentFields) { + for (RexNode r : fields) { + if (r.getKind() == AS) { + RexCall as = (RexCall) r; + if (as.getOperands().get(0) instanceof RexInputRef ref) { + String name = ((RexLiteral) as.getOperands().get(1)).getValueAs(String.class); + if (!name.equals(currentFields.get(ref.getIndex()))) { + return true; + } + } + } + } + return false; + } + private boolean isSingleAllFieldsProject(Project node) { return node.getProjectList().size() == 1 && node.getProjectList().getFirst() instanceof AllFields; From 7054bb59cef23ed37b11016fc953565d3a1c9312 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:58:07 -0700 Subject: [PATCH 09/36] Skip AnalyticsEngineCompatIT when analytics-engine is absent (#5510) AnalyticsEngineCompatIT is a coexistence smoke test meant to run only in the dedicated :integ-test:analyticsEngineCompatIT task, which bundles the analytics-engine plugin stack. The distribution integ-test pipeline, however, scans every *IT class against the built distribution and runs it with the security plugin enabled and without analytics-engine. There it fails during REST client init with "ConnectionClosedException: Connection closed by peer", because the test extends a bare OpenSearchRestTestCase that speaks plain HTTP to the TLS-secured port. A build.gradle exclude does not cover that pipeline. Make the test self-inert regardless of how it is discovered: - Extend OpenSearchSQLRestTestCase so the REST client honours the https/user/ password system properties of a secured cluster. - Skip via an assumption when the analytics-engine plugin is not installed, so a build without the plugin reports the test as skipped rather than failed. Signed-off-by: Kai Huang --- .../sql/plugin/AnalyticsEngineCompatIT.java | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index 5cd89fa7cd9..f6ec903c395 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,17 +5,55 @@ package org.opensearch.sql.plugin; +import static org.junit.Assume.assumeTrue; + +import java.io.IOException; +import org.junit.Before; import org.junit.Test; -import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.legacy.OpenSearchSQLRestTestCase; +import org.opensearch.sql.legacy.TestUtils; /** * Smoke test: verifies that opensearch-sql loads cleanly alongside arrow-flight-rpc and * analytics-engine. A successful cluster start is the only assertion — no sql-specific logic runs. + * + *

This test is only meaningful when the analytics-engine plugin is installed; the dedicated + * {@code :integ-test:analyticsEngineCompatIT} Gradle task bundles the plugin stack for exactly that + * purpose. Other lanes can still discover this class — notably the distribution integ-test + * pipeline, which scans every {@code *IT} class against the built distribution and may run with the + * security plugin enabled and without analytics-engine. A {@code build.gradle} exclude does not + * protect against that pipeline, so the test guards itself instead: + * + *

    + *
  • It extends {@link OpenSearchSQLRestTestCase} so the REST client honours the {@code https}, + * {@code user}, and {@code password} system properties of a secured cluster (a bare {@code + * OpenSearchRestTestCase} speaks plain HTTP and gets its connection closed during client init + * on a TLS-secured port). + *
  • It skips via an assumption when analytics-engine is absent, so a build without the plugin + * reports the test as skipped rather than failed. + *
*/ -public class AnalyticsEngineCompatIT extends OpenSearchRestTestCase { +public class AnalyticsEngineCompatIT extends OpenSearchSQLRestTestCase { + + /** + * Skips the suite unless the analytics-engine plugin is installed. Runs after the framework has + * established the (security-aware) REST client, so the lookup itself succeeds on both plain and + * secured clusters. + */ + @Before + public void requireAnalyticsEngine() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/plugins?h=component")); + String installedPlugins = TestUtils.getResponseBody(response, true); + assumeTrue( + "analytics-engine plugin not installed — skipping coexistence smoke test", + installedPlugins.contains("analytics-engine")); + } @Test public void testClusterStarted() { - // If the cluster booted, all three plugins loaded without classloader errors. + // If the cluster booted with analytics-engine present, all plugins loaded without classloader + // errors. The assumption above guarantees we only assert this where it is meaningful. } } From 2c4215f300691cf12da37bcbb27b25ba414cf971 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Thu, 4 Jun 2026 15:52:09 -0700 Subject: [PATCH 10/36] Skip PUT+DELETE doc-fixture tests on the analytics-engine storage matrix (#5476) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These 10 PPL/Calcite integration tests mutate the shared test index by issuing PUT //_doc/ + DELETE //_doc/ around the query under test. When the suite runs with -Dtests.analytics.parquet_indices=true, test indices are backed by DataFormatAwareEngine, whose delete() and prepareDelete() deliberately throw "delete operation not supported." — parquet/composite storage is append-only by design. The tests pass under the Lucene-backed default. Skipping them via Assume.assumeFalse(isAnalyticsParquetIndicesEnabled()) follows the precedent already established in StatsCommandIT and CalciteAnalyticsDatetimeWireFormatIT, and prevents the spurious bucket-22 "delete operation not supported" failures in the AE-storage report without changing any assertion or test behavior on the default path. Signed-off-by: Jialiang Liang --- .../calcite/remote/CalciteExpandCommandIT.java | 9 +++++++++ .../remote/CalciteStreamstatsCommandIT.java | 13 +++++++++++++ .../opensearch/sql/ppl/DateTimeFunctionIT.java | 5 +++++ .../org/opensearch/sql/ppl/SearchCommandIT.java | 17 +++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java index f7e994223c6..a7aac6765b5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ARRAY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -295,6 +296,10 @@ public void testExpandWithEval() throws Exception { @Test public void testExpandEmptyArray() throws Exception { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 6; Request insertRequest = new Request( @@ -324,6 +329,10 @@ public void testExpandEmptyArray() throws Exception { @Test public void testExpandOnNullField() throws Exception { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 6; Request insertRequest = new Request( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java index 77daf6dce48..27200078562 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.*; @@ -507,6 +508,10 @@ public void testStreamstatsCurrentAndWindowWithNull() throws IOException { @Test public void testStreamstatsGlobal() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 5; Request insertRequest = new Request( @@ -666,6 +671,10 @@ public void testStreamstatsGlobalWithNullBucket() throws IOException { @Test public void testStreamstatsReset() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 5; Request insertRequest = new Request( @@ -934,6 +943,10 @@ public void testMultipleStreamstatsWithNull1() throws IOException { @Test public void testMultipleStreamstatsWithNull2() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 5; Request insertRequest = new Request( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index cbfe3c84464..aa5b7758446 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -1565,6 +1566,10 @@ public void testExtract() throws IOException { @Test public void testCompareAgainstUTCDate() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); String isoTimestamp = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'")); String pplTimestamp = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java index b8e5b2a5722..c5360a5dfa0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.columnName; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -1053,6 +1054,10 @@ public void testSearchWithNumericTimeRange() throws IOException { @Test public void testSearchTimeModifierWithSnappedWeek() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); // Test whether alignment to weekday works final int docId = 101; @@ -1141,6 +1146,10 @@ public void testSearchTimeModifierWithSnappedWeek() throws IOException { @Test public void testSearchWithRelativeTimeModifiers() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 101; LocalDateTime currentTime = LocalDateTime.now(ZoneOffset.UTC); @@ -1189,6 +1198,10 @@ public void testSearchWithRelativeTimeModifiers() throws IOException { @Test public void testSearchWithTimeUnitSnapping() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 101; LocalDateTime currentHour = LocalDateTime.now(ZoneOffset.UTC).truncatedTo(ChronoUnit.HOURS); @@ -1237,6 +1250,10 @@ public void testSearchWithTimeUnitSnapping() throws IOException { @Test public void testSearchWithQuarterlyModifiers() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 101; LocalDateTime currentQuarter = From cf14abae26bf2d16ead8f5fa82e9695fd09d02c0 Mon Sep 17 00:00:00 2001 From: Lantao Jin Date: Fri, 5 Jun 2026 15:46:57 +0800 Subject: [PATCH 11/36] Coerce temporal operands in IN and BETWEEN when leastRestrictive finds no common type (#5513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Coerce temporal operands in IN and BETWEEN when leastRestrictive finds no common type visitIn and visitBetween call leastRestrictive directly instead of going through the CoercionUtils path that comparison operators use. leastRestrictive returns no common type when temporal operands are represented differently — e.g. a standard Calcite TIMESTAMP field against EXPR_DATE UDT bounds/values (`ts between date('...') and date('...')`, `ts in (DATE '...', ...)`) — so both predicates rejected such queries with "expression types are incompatible" even though comparisons coerce the same mix. Both now fall back to CoercionUtils.widenArguments (the comparison-operator path) when leastRestrictive yields null, scoped to all-temporal operands so genuinely incompatible mixes (e.g. `age between '35' and 38.5`) still raise SemanticCheckException. Enables the previously-undiscovered testDateBetween (was missing its @Test annotation) and adds testDateIn plus a CoercionUtils unit test covering the plain-TIMESTAMP + EXPR_DATE mix. Signed-off-by: Lantao Jin * Compare temporal IN/NOT IN values in the field's timestamp domain instead of string-collapsing them Signed-off-by: Lantao Jin --------- Signed-off-by: Lantao Jin --- .../sql/calcite/CalciteRexNodeVisitor.java | 80 +++++++++++++++---- .../function/CoercionUtilsTest.java | 35 ++++++++ .../sql/calcite/remote/CalcitePPLBasicIT.java | 52 ++++++++++++ .../opensearch/request/PredicateAnalyzer.java | 43 +++++++++- 4 files changed, 193 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 830b40d2551..1d473b168e4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nullable; @@ -84,11 +85,13 @@ import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.SubsearchUtils; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.CoercionUtils; import org.opensearch.sql.expression.function.PPLFuncImpTable; @RequiredArgsConstructor @@ -225,24 +228,41 @@ public RexNode visitIn(In node, CalcitePlanContext context) { final RexNode field = analyze(node.getField(), context); final List valueList = node.getValueList().stream().map(value -> analyze(value, context)).toList(); + // When the field is a temporal type, do NOT use leastRestrictive + rexBuilder.makeIn. For a + // temporal field tested against string/date literals, leastRestrictive collapses the common + // type to VARCHAR (the EXPR_DATE / EXPR_TIMESTAMP UDTs are VARCHAR-backed), so makeIn casts the + // field DOWN to VARCHAR and string-compares mismatched renderings (e.g. '2018-06-23 00:00:00' + // against '2018-06-23') — silently matching nothing. Rewrite the membership test as an OR of + // PPL `=` comparisons, the same temporal-aware comparison path visitCompare takes for `=`, so + // each value is coerced to the field's timestamp domain before comparison. + ExprType fieldExprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(field.getType()); + if (TEMPORAL_TYPES.contains(fieldExprType)) { + List equalities = + valueList.stream() + .map(value -> PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, "=", field, value)) + .toList(); + return context.relBuilder.or(equalities); + } final List dataTypes = - new java.util.ArrayList<>(valueList.stream().map(RexNode::getType).toList()); + new ArrayList<>(valueList.stream().map(RexNode::getType).toList()); dataTypes.add(field.getType()); RelDataType commonType = context.rexBuilder.getTypeFactory().leastRestrictive(dataTypes); if (commonType != null) { List newValueList = valueList.stream().map(value -> context.rexBuilder.makeCast(commonType, value)).toList(); return context.rexBuilder.makeIn(field, newValueList); - } else { - List exprTypes = - dataTypes.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); - throw new SemanticCheckException( - StringUtils.format( - "In expression types are incompatible: fields type %s, values type %s", - exprTypes.getLast(), exprTypes.subList(0, exprTypes.size() - 1))); } + List exprTypes = + dataTypes.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); + throw new SemanticCheckException( + StringUtils.format( + "In expression types are incompatible: fields type %s, values type %s", + exprTypes.getLast(), exprTypes.subList(0, exprTypes.size() - 1))); } + private static final Set TEMPORAL_TYPES = + Set.of(ExprCoreType.DATE, ExprCoreType.TIME, ExprCoreType.TIMESTAMP); + @Override public RexNode visitCompare(Compare node, CalcitePlanContext context) { RexNode left = analyze(node.getLeft(), context); @@ -258,6 +278,25 @@ public RexNode visitCompare(Compare node, CalcitePlanContext context) { return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, op, left, right); } + /** + * Widens a set of operands to a common temporal type when, and only when, every operand is a + * temporal type (DATE / TIME / TIMESTAMP), including the EXPR_DATE / EXPR_TIME / EXPR_TIMESTAMP + * UDTs. Returns {@code null} otherwise so non-temporal incompatible mixes still fail the type + * check. The widening reuses {@link CoercionUtils#widenArguments} — the same path comparison + * operators take — which resolves DATE / TIME to TIMESTAMP via the shared widening graph. + */ + private static @Nullable List widenTemporalOperands( + CalcitePlanContext context, List operands) { + boolean allTemporal = + operands.stream() + .map(node -> OpenSearchTypeFactory.convertRelDataTypeToExprType(node.getType())) + .allMatch(TEMPORAL_TYPES::contains); + if (!allTemporal) { + return null; + } + return CoercionUtils.widenArguments(context.rexBuilder, operands); + } + @Override public RexNode visitBetween(Between node, CalcitePlanContext context) { RexNode value = analyze(node.getValue(), context); @@ -268,12 +307,25 @@ public RexNode visitBetween(Between node, CalcitePlanContext context) { lowerBound = context.rexBuilder.makeCast(commonType, lowerBound); upperBound = context.rexBuilder.makeCast(commonType, upperBound); } else { - throw new SemanticCheckException( - StringUtils.format( - "BETWEEN expression types are incompatible: [%s, %s, %s]", - OpenSearchTypeFactory.convertRelDataTypeToExprType(value.getType()), - OpenSearchTypeFactory.convertRelDataTypeToExprType(lowerBound.getType()), - OpenSearchTypeFactory.convertRelDataTypeToExprType(upperBound.getType()))); + // leastRestrictive() has no common type for mixed temporal representations — e.g. a standard + // Calcite TIMESTAMP field compared against EXPR_DATE UDT bounds (`ts between date('...') and + // date('...')`). Comparison operators coerce these through CoercionUtils; BETWEEN calls + // leastRestrictive directly and would otherwise reject them. Fall back to the same temporal + // widening, scoped to all-temporal operands so genuinely incompatible mixes (e.g. + // `age between '35' and 38.5`) still raise SemanticCheckException. + List widened = + widenTemporalOperands(context, List.of(value, lowerBound, upperBound)); + if (widened == null) { + throw new SemanticCheckException( + StringUtils.format( + "BETWEEN expression types are incompatible: [%s, %s, %s]", + OpenSearchTypeFactory.convertRelDataTypeToExprType(value.getType()), + OpenSearchTypeFactory.convertRelDataTypeToExprType(lowerBound.getType()), + OpenSearchTypeFactory.convertRelDataTypeToExprType(upperBound.getType()))); + } + value = widened.get(0); + lowerBound = widened.get(1); + upperBound = widened.get(2); } return context.relBuilder.between(value, lowerBound, upperBound); } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java index 373881e8543..5533fd90915 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java @@ -17,6 +17,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -51,6 +52,40 @@ public void findCommonWidestType( expectedCommonType, CoercionUtils.resolveCommonType(left, right).orElseGet(() -> null)); } + @Test + void widenArgumentsUnifiesPlainTimestampWithDateUdtBounds() { + // Reproduces the BETWEEN/IN operand set seen on a non-UDT path (e.g. the analytics engine's + // parquet scan): a standard Calcite TIMESTAMP field against EXPR_DATE UDT bounds. + // leastRestrictive + // returns no common type for this mix, but the temporal widening used by comparison operators + // must resolve all three to a single timestamp type so the predicate can run. + RexNode plainTimestampField = + REX_BUILDER.makeInputRef( + OpenSearchTypeFactory.TYPE_FACTORY.createTypeWithNullability( + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), true), + 0); + RexNode dateLower = + REX_BUILDER.makeNullLiteral( + OpenSearchTypeFactory.TYPE_FACTORY.createUDT( + OpenSearchTypeFactory.ExprUDT.EXPR_DATE, true)); + RexNode dateUpper = + REX_BUILDER.makeNullLiteral( + OpenSearchTypeFactory.TYPE_FACTORY.createUDT( + OpenSearchTypeFactory.ExprUDT.EXPR_DATE, true)); + + List widened = + CoercionUtils.widenArguments( + REX_BUILDER, List.of(plainTimestampField, dateLower, dateUpper)); + + assertNotNull(widened); + assertEquals(3, widened.size()); + for (RexNode node : widened) { + assertEquals( + ExprCoreType.TIMESTAMP, + OpenSearchTypeFactory.convertRelDataTypeToExprType(node.getType())); + } + } + @Test void castArgumentsReturnsExactMatchWhenAvailable() { PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(INTEGER), List.of(DOUBLE))); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index 4c2c8176690..27d971a4615 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -500,7 +500,10 @@ public void testNotBetween3() throws IOException { verifyDataRows(actual, rows("Hattie", 36), rows("Elinor", 36)); } + @Test public void testDateBetween() throws IOException { + // birthdate is a TIMESTAMP-typed field; the bounds are DATE literals. BETWEEN must coerce the + // mixed temporal operands rather than reject them with "expression types are incompatible". JSONObject actual = executeQuery( String.format( @@ -512,6 +515,55 @@ public void testDateBetween() throws IOException { actual, rows("Nanette", "2018-06-23 00:00:00"), rows("Elinor", "2018-06-27 00:00:00")); } + @Test + public void testDateIn() throws IOException { + // birthdate is a TIMESTAMP-typed field; the IN values are DATE literals. visitIn must compare + // each value in the field's temporal domain (via PPL `=`) rather than letting leastRestrictive + // collapse the common type to VARCHAR and string-compare mismatched renderings — which silently + // matched nothing without pushdown. See visitIn in CalciteRexNodeVisitor. + JSONObject actual = + executeQuery( + String.format( + "source=%s | where birthdate in (DATE '2018-06-23', DATE '2018-06-27') |" + + " fields firstname, birthdate", + TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string"), schema("birthdate", "timestamp")); + verifyDataRows( + actual, rows("Nanette", "2018-06-23 00:00:00"), rows("Elinor", "2018-06-27 00:00:00")); + } + + @Test + public void testDateNotIn() throws IOException { + // Complement of testDateIn: NOT IN over a temporal field. Exercises the complemented-points + // pushdown branch, which must also format timestamp values rather than emit a flat terms query. + JSONObject actual = + executeQuery( + String.format( + "source=%s | where birthdate not in (DATE '2018-06-23', DATE '2018-06-27') |" + + " fields firstname | sort firstname", + TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string")); + verifyDataRows( + actual, + rows("Amber JOHnny"), + rows("Dale"), + rows("Dillard"), + rows("Hattie"), + rows("Virginia")); + } + + @Test + public void testIn() throws IOException { + // Non-temporal IN keeps the leastRestrictive + makeIn path; guards against the temporal-field + // special-case in visitIn regressing membership tests over ordinary columns. + JSONObject actual = + executeQuery( + String.format( + "source=%s | where age in (32, 36) | fields firstname, age", TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string"), schema("age", "int")); + verifyDataRows(actual, rows("Amber JOHnny", 32), rows("Hattie", 36), rows("Elinor", 36)); + } + @Test public void testXor() throws IOException { JSONObject result = diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 6389fb28395..29f44adf086 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -784,16 +784,29 @@ private QueryExpression like(RexCall call) { private static QueryExpression constructQueryExpressionForSearch( RexCall call, SwapResult pair) { + boolean isTimeStamp = + (pair.getKey() instanceof NamedFieldExpression namedField) + && namedField.isTimeStampType(); if (isSearchWithComplementedPoints(call)) { + // A NOT IN over a timestamp field cannot use a flat terms query: timestamp values must be + // normalized + formatted (see in()/termsQuery, which does neither). Decompose into the + // complement of an OR of formatted per-point range queries, mirroring the range branch. + if (isTimeStamp) { + return pointsAsTimestampOr(call, pair).not(); + } return QueryExpression.create(pair.getKey()).notIn(pair.getValue()); } else if (isSearchWithPoints(call)) { + // IN over a timestamp field: the flat terms query in in() emits unformatted values like + // '2018-06-23 00:00:00' that the date field cannot parse. Decompose into an OR of formatted + // per-point range queries (equals(point, true) builds gte/lte + format="date_time"), the + // same path the range branch below and visitCompare's `=` already take. + if (isTimeStamp) { + return pointsAsTimestampOr(call, pair); + } return QueryExpression.create(pair.getKey()).in(pair.getValue()); } else { Sarg sarg = pair.getValue().literal.getValueAs(Sarg.class); Set> rangeSet = requireNonNull(sarg).rangeSet.asRanges(); - boolean isTimeStamp = - (pair.getKey() instanceof NamedFieldExpression namedField) - && namedField.isTimeStampType(); List queryExpressions = rangeSet.stream() .map( @@ -811,6 +824,30 @@ private static QueryExpression constructQueryExpressionForSearch( } } + /** + * Builds an OR of per-point {@code equals(point, true)} query expressions for a points-only + * Sarg over a timestamp field. Each point becomes a {@code gte/lte} range query with {@code + * format="date_time"}, so the timestamp value is normalized to the date field's accepted format + * — unlike a flat terms query, which emits unformatted values the date field rejects. + */ + private static QueryExpression pointsAsTimestampOr(RexCall call, SwapResult pair) { + RexLiteral literal = (RexLiteral) call.getOperands().get(1); + Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); + Set> ranges = + (sarg.isComplementedPoints() ? sarg.negate() : sarg).rangeSet.asRanges(); + List queryExpressions = + ranges.stream() + .map( + range -> + QueryExpression.create(pair.getKey()) + .equals(sargPointValue(range.lowerEndpoint()), true)) + .toList(); + if (queryExpressions.size() == 1) { + return queryExpressions.getFirst(); + } + return CompoundQueryExpression.or(queryExpressions.toArray(new QueryExpression[0])); + } + private QueryExpression andOr(RexCall call) { QueryExpression[] expressions = new QueryExpression[call.getOperands().size()]; PredicateAnalyzerException firstError = null; From e5de8f6a77a3280f058d9097346f2373ae20dc9d Mon Sep 17 00:00:00 2001 From: ryan-gh-bot Date: Mon, 8 Jun 2026 09:55:24 -0700 Subject: [PATCH 12/36] [BugFix] Restore dedup pushdown when combined with WHERE clause (#5482) (#5488) * [BugFix] Restore dedup pushdown when combined with WHERE clause (#5482) Run PPLSimplifyDedupRule before FilterMergeRule in the HEP optimizer so the bucket-non-null filter PPL emits for dedup is matched as-is. With the previous order, an upstream user where filter sat adjacent to the bucket-non-null filter; FilterMergeRule fired first and merged them into a conjunction that no longer satisfied PPLSimplifyDedupRule's operand predicate, defeating dedup pushdown to the shard. Use sequential addRuleInstance phases for explicit ordering rather than addRuleCollection, which is documented as non-deterministic in firing order. Adds two regression tests in CalcitePPLDedupTest: one that asserts LogicalDedup is produced under the fixed order, and one that pins the buggy behavior under the swapped order. Signed-off-by: ryan-gh-bot * [BugFix] Drop issue-link reference from regression-test JavaDoc (#5488) Per maintainer review feedback, the regression-test JavaDoc for testDedupAfterWhereProducesLogicalDedup mentioned the originating issue URL. The remaining JavaDoc paragraphs already describe the bug shape and the rule-ordering invariant, so the explicit issue link is unnecessary noise. Signed-off-by: ryan-gh-bot * [BugFix] Make dedup simplify operand order-independent (#5488) Address review feedback on #5488: extend mayBeFilterFromBucketNonNull to accept the merged conjunction shape FilterMergeRule produces, so PPLSimplifyDedupRule fires regardless of whether FilterMergeRule has already merged the user where clause into the bucket-non-null filter. PPLSimplifyDedupRule.apply now splits the bottom filter into IS_NOT_NULL conjuncts on partition keys (absorbed into LogicalDedup semantics) and any remaining conjuncts (preserved as a separate filter below the new LogicalDedup), so a user predicate that was folded in is no longer dropped. With the operand predicate order-independent, the HEP rule order is no longer a load-bearing invariant. Revert the addRuleCollection -> addRuleInstance change in CalciteToolsHelper.HEP_PROGRAM that the previous patch introduced. Replace the regression test that pinned the buggy rule order with one that asserts the user-visible contract: with where preceding dedup, a LogicalDedup is produced and the user predicate is preserved regardless of which order FilterMergeRule and PPLSimplifyDedupRule fire. Signed-off-by: ryan-gh-bot * Address review comments on #5488 Per @penghuo review: PlanUtils.java - Revert mayBeFilterFromBucketNonNull to the original ternary form; drop the early-return refactor (no behavior change, just cleaner). - Drop the !rexCall.getOperands().isEmpty() guard before .get(0): IS NOT NULL is always unary in Calcite, so the check is dead. - Trim the JavaDoc to the essentials (un-merged vs merged-AND shape; concrete partition-key match happens in PPLSimplifyDedupRule#apply). - Promote isNotNullOnRef from package-private to public so the dedup rule can reuse it from a different package. PPLSimplifyDedupRule.java - isNotNullOnPartitionKey now delegates the IS NOT NULL($ref) structural check to PlanUtils.isNotNullOnRef and adds the partition-key index check on top. CalciteExplainIT.java - Add testDedupAfterWherePushDown: an end-to-end regression that runs the shape `... | where | dedup ` and asserts (a) LogicalDedup appears in the explain output (PPLSimplifyDedupRule fired even after FilterMergeRule had a chance to merge the two filters), and (b) EnumerableWindow does NOT appear (the in-memory ROW_NUMBER fallback the bug caused is gone). Signed-off-by: Jialiang Liang * Push user where filter into scan when blocking dedup pushdown PPLSimplifyDedupRule correctly produces Dedup -> Filter(user where) -> Scan when a `where` precedes `dedup`. The Filter between Dedup and Scan blocks DedupPushdownRule's strict Dedup -> Project -> Scan operand chain, so Volcano falls back to PPLDedupConvertRule and the plan ends up with an in-memory ROW_NUMBER window instead of the pushed-down composite + top_hits aggregation. Add a WITH_FILTER operand variant to DedupPushdownRule that matches Dedup -> Filter -> Scan, pushes the filter into the scan, then runs the standard apply() on the resulting Dedup -> Project -> Scan shape. Signed-off-by: Jialiang Liang * Bail when filter is only partially pushable pushDownFilter returns a Filter (not a CalciteLogicalIndexScan) when the predicate analyzer can only partially push the condition. The previous cast would have thrown ClassCastException in that case. Use an instanceof-pattern check so the rule bails out cleanly and leaves the plan untouched, letting other rules handle the residual. Also drop a stale issue-link reference from a test comment. Signed-off-by: Jialiang Liang * Apply spotless formatting to dedup unit test Reflowed the JavaDoc on testDedupAfterWhereProducesLogicalDedupWithProductionHepProgram to match Google Java Format's preferred line break, fixing the spotlessJavaCheck violation that failed the unit-test matrix on CI. Signed-off-by: Jialiang Liang --------- Signed-off-by: ryan-gh-bot Signed-off-by: Jialiang Liang Co-authored-by: ryan-gh-bot Co-authored-by: Jialiang Liang --- .../plan/rule/PPLSimplifyDedupRule.java | 45 ++++++ .../sql/calcite/utils/PlanUtils.java | 10 +- .../sql/calcite/remote/CalciteExplainIT.java | 23 +++ .../planner/rules/DedupPushdownRule.java | 40 ++++++ .../planner/rules/OpenSearchIndexRules.java | 3 + .../ppl/calcite/CalcitePPLAbstractTest.java | 14 ++ .../sql/ppl/calcite/CalcitePPLDedupTest.java | 131 ++++++++++++++++++ 7 files changed, 264 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java index 11eabfd483c..676b1e9a776 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java @@ -6,11 +6,14 @@ package org.opensearch.sql.calcite.plan.rule; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelRule; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; @@ -22,6 +25,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexWindow; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; @@ -115,8 +119,41 @@ protected void apply( RelCollation inputCollation = extractCollationFromWindow(windows.get(0)); + // Split the bucket-non-null filter into two parts: + // 1) IS_NOT_NULL conjuncts on a partition key — these are the bucket-non-null guards PPL + // emits as part of the dedup pattern; LogicalDedup absorbs their semantics. + // 2) Everything else — for example, a user `where` predicate that FilterMergeRule may + // have folded into the same conjunction, or a user IS_NOT_NULL filter on a non-partition + // column. These must be preserved as a separate filter below the new LogicalDedup so + // user-visible behavior is unchanged regardless of whether FilterMergeRule fired. + Set partitionKeyIndices = new HashSet<>(); + for (RexNode key : dedupColumns) { + if (key instanceof RexInputRef ref) { + partitionKeyIndices.add(ref.getIndex()); + } + } + List bucketNonNullConjuncts = new ArrayList<>(); + List remainingConjuncts = new ArrayList<>(); + for (RexNode conjunct : RelOptUtil.conjunctions(bucketNonNullFilter.getCondition())) { + if (isNotNullOnPartitionKey(conjunct, partitionKeyIndices)) { + bucketNonNullConjuncts.add(conjunct); + } else { + remainingConjuncts.add(conjunct); + } + } + // Defensive: if no IS_NOT_NULL conjunct on a partition key is present, this filter is not + // actually a bucket-non-null filter — bail out without transforming. The loose operand + // predicate may have matched on an unrelated AND that happens to contain an IS_NOT_NULL on + // some other ref. + if (bucketNonNullConjuncts.isEmpty()) { + return; + } + RelBuilder relBuilder = call.builder(); relBuilder.push(bucketNonNullFilter.getInput()); + if (!remainingConjuncts.isEmpty()) { + relBuilder.filter(RexUtil.composeConjunction(relBuilder.getRexBuilder(), remainingConjuncts)); + } List> targetProjections = projectWithWindow.getNamedProjects().stream() .filter(p -> !p.getKey().isA(SqlKind.ROW_NUMBER)) @@ -134,6 +171,14 @@ protected void apply( call.transformTo(relBuilder.build()); } + private static boolean isNotNullOnPartitionKey(RexNode rex, Set partitionKeyIndices) { + if (!PlanUtils.isNotNullOnRef(rex)) { + return false; + } + RexInputRef ref = (RexInputRef) ((RexCall) rex).getOperands().get(0); + return partitionKeyIndices.contains(ref.getIndex()); + } + private static @Nullable RelCollation extractCollationFromWindow(RexWindow window) { if (window.orderKeys.isEmpty()) { return null; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 84832071857..f899f747421 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -788,15 +788,21 @@ static Mapping mapping(List rexNodes, RelDataType schema) { return Mappings.target(getSelectColumns(rexNodes), schema.getFieldCount()); } + /** + * Accepts the un-merged {@code IS NOT NULL($ref)} shape and the merged-{@code AND} shape that + * {@link org.apache.calcite.rel.rules.FilterMergeRule} produces when a user {@code where} + * precedes the dedup. The concrete partition-key match — and the split-out of any user predicate + * folded into the AND — happens in {@link PPLSimplifyDedupRule#apply}. + */ static boolean mayBeFilterFromBucketNonNull(LogicalFilter filter) { RexNode condition = filter.getCondition(); return isNotNullOnRef(condition) || (condition instanceof RexCall rexCall && rexCall.getOperator().equals(SqlStdOperatorTable.AND) - && rexCall.getOperands().stream().allMatch(PlanUtils::isNotNullOnRef)); + && rexCall.getOperands().stream().anyMatch(PlanUtils::isNotNullOnRef)); } - private static boolean isNotNullOnRef(RexNode rex) { + public static boolean isNotNullOnRef(RexNode rex) { return rex instanceof RexCall rexCall && rexCall.isA(SqlKind.IS_NOT_NULL) && rexCall.getOperands().get(0) instanceof RexInputRef; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 8ad0be5cc88..d25631c5ace 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2153,6 +2153,29 @@ public void testTransposeExplain() throws IOException { + "| transpose 4 column_name='column_names'")); } + /** + * With a user {@code where} clause preceding {@code dedup}, the physical plan must push both the + * filter and the dedup-as-aggregation into the OpenSearch scan, not fall back to an in-memory + * {@code ROW_NUMBER} window above a row-fetching scan. + */ + @Test + public void testDedupAfterWherePushDown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String result = + explainQueryToString( + "source=opensearch-sql_test_index_account | where age > 25 | dedup gender"); + assertTrue( + "Expected user where filter pushed down to the scan:\n" + result, + result.contains("FILTER->>($8, 25)")); + assertTrue( + "Expected dedup pushed down as AGGREGATION (composite + top_hits):\n" + result, + result.contains("AGGREGATION->")); + assertFalse( + "Unexpected EnumerableWindow — dedup fell back to the in-memory ROW_NUMBER form:\n" + + result, + result.contains("EnumerableWindow")); + } + public void testComplexDedup() throws IOException { enabledOnlyWhenPushdownIsEnabled(); String expected = loadExpectedPlan("explain_dedup_complex1.yaml"); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java index d37957d4a9f..648888ce9b7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java @@ -17,6 +17,7 @@ import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.rules.SubstitutionRule; import org.apache.calcite.rex.RexInputRef; @@ -47,6 +48,24 @@ protected DedupPushdownRule(Config config) { @Override protected void onMatchImpl(RelOptRuleCall call) { final LogicalDedup logicalDedup = call.rel(0); + if (call.rels[1] instanceof LogicalFilter filter) { + // Push the filter into the scan, then synthesize an identity project so the standard + // apply() can run on the resulting Dedup → Project → Scan shape. If the filter is only + // partially pushable, pushDownFilter returns a Filter wrapping the residual condition over + // the new scan; we can't strip a residual filter without breaking semantics, so bail. + final CalciteLogicalIndexScan scan = call.rel(2); + if (!(scan.pushDownFilter(filter) instanceof CalciteLogicalIndexScan newScan)) { + return; + } + RelBuilder relBuilder = call.builder(); + relBuilder.push(newScan); + // force=true so the identity project is materialized (apply() requires a LogicalProject) + relBuilder.project( + relBuilder.fields(), newScan.getRowType().getFieldNames(), /* force= */ true); + LogicalProject identityProject = (LogicalProject) relBuilder.build(); + apply(call, logicalDedup, identityProject, newScan); + return; + } final LogicalProject projectWithExpr = call.rel(1); final CalciteLogicalIndexScan scan = call.rel(2); apply(call, logicalDedup, projectWithExpr, scan); @@ -226,6 +245,27 @@ public interface Config extends OpenSearchRuleConfig { .predicate(Config::tableScanChecker) .noInputs()))); + // +- LogicalDedup + // +- LogicalFilter (e.g. a `where` predicate preserved below dedup by + // +- CalciteLogicalIndexScan PPLSimplifyDedupRule when an upstream `where` is folded + // into the bucket-non-null filter) + Config WITH_FILTER = + ImmutableDedupPushdownRule.Config.builder() + .build() + .withDescription("Dedup-to-Aggregate-WithFilter") + .withOperandSupplier( + b0 -> + b0.operand(LogicalDedup.class) + .predicate(dedup -> !dedup.getKeepEmpty()) + .oneInput( + b1 -> + b1.operand(LogicalFilter.class) + .oneInput( + b2 -> + b2.operand(CalciteLogicalIndexScan.class) + .predicate(Config::tableScanChecker) + .noInputs()))); + /** * Project must be not pushed since the name of expression would lose after project pushed. E.g. * in query "eval new_a = a + 1 | dedup b", the "new_a" will lose. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 0068f445ce7..3c8508cc455 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -50,6 +50,8 @@ public class OpenSearchIndexRules { SortIndexScanRule.Config.DEFAULT.toRule(); private static final DedupPushdownRule DEDUP_PUSH_DOWN = DedupPushdownRule.Config.DEFAULT.toRule(); + private static final DedupPushdownRule DEDUP_PUSH_DOWN_WITH_FILTER = + DedupPushdownRule.Config.WITH_FILTER.toRule(); private static final SortProjectExprTransposeRule SORT_PROJECT_EXPR_TRANSPOSE = SortProjectExprTransposeRule.Config.DEFAULT.toRule(); private static final ExpandCollationOnProjectExprRule EXPAND_COLLATION_ON_PROJECT_EXPR = @@ -75,6 +77,7 @@ public class OpenSearchIndexRules { LIMIT_INDEX_SCAN, SORT_INDEX_SCAN, DEDUP_PUSH_DOWN, + DEDUP_PUSH_DOWN_WITH_FILTER, SORT_PROJECT_EXPR_TRANSPOSE, SORT_AGGREGATION_METRICS_RULE, RARE_TOP_PUSH_DOWN, diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java index ab07cd9b5c1..ec3b57d02cc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java @@ -110,6 +110,20 @@ public RelNode getRelNode(String ppl) { return root; } + /** + * Get the root RelNode of the given PPL query without merging adjacent filters. Useful for + * regression tests that need to exercise rule ordering against the un-merged shape that PPL + * actually emits to the production HEP optimizer. + */ + public RelNode getRelNodeRaw(String ppl) { + CalcitePlanContext context = createBuilderContext(); + Query query = (Query) plan(pplParser, ppl); + planTransformer.analyze(query.getPlan(), context); + RelNode root = context.relBuilder.build(); + System.out.println(root.explain()); + return root; + } + private RelNode mergeAdjacentFilters(RelNode relNode) { HepProgram program = new HepProgramBuilder().addRuleInstance(FilterMergeRule.Config.DEFAULT.toRule()).build(); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java index ca1a789b0f4..5f32c1b85bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java @@ -5,9 +5,15 @@ package org.opensearch.sql.ppl.calcite; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rules.FilterMergeRule; import org.apache.calcite.test.CalciteAssert; +import org.junit.Assert; import org.junit.Test; +import org.opensearch.sql.calcite.plan.rule.PPLSimplifyDedupRule; public class CalcitePPLDedupTest extends CalcitePPLAbstractTest { @@ -353,4 +359,129 @@ public void testSortFieldProjectedAwayBeforeDedup() { + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } + + /** + * When a user {@code where} precedes {@code dedup}, the user filter sits adjacent to the + * bucket-non-null filter that PPL emits for the dedup pattern. {@link PPLSimplifyDedupRule} must + * fold the dedup pattern into a {@link org.opensearch.sql.calcite.plan.rel.LogicalDedup} so that + * {@code DedupPushdownRule} can match it; otherwise dedup falls through to the in-memory {@code + * ROW_NUMBER} window form, defeating dedup pushdown to the shard. The user predicate must be + * preserved as a separate filter below the new {@code LogicalDedup}. + */ + @Test + public void testDedupAfterWhereProducesLogicalDedup() { + String ppl = "source=EMP | where SAL > 1000 | dedup DEPTNO"; + RelNode raw = getRelNodeRaw(ppl); + + // Sanity: the un-merged plan has both the bucket-non-null filter and the user where filter + // adjacent to each other above the scan — exactly the shape that triggered the original bug. + String rawExplain = raw.explain(); + Assert.assertTrue( + "Raw plan should contain the bucket-non-null filter:\n" + rawExplain, + rawExplain.contains("IS NOT NULL")); + Assert.assertTrue( + "Raw plan should contain the user where filter:\n" + rawExplain, + rawExplain.contains(">($5, 1000)")); + Assert.assertTrue( + "Raw plan should contain ROW_NUMBER prior to simplification:\n" + rawExplain, + rawExplain.contains("ROW_NUMBER")); + + // Apply rules in the order PPLSimplifyDedupRule -> FilterMergeRule. + HepProgram program = + new HepProgramBuilder() + .addRuleInstance(PPLSimplifyDedupRule.DEDUP_SIMPLIFY_RULE) + .addRuleInstance(FilterMergeRule.Config.DEFAULT.toRule()) + .build(); + RelNode optimized = runHepPlanner(raw, program); + + String optimizedExplain = optimized.explain(); + Assert.assertTrue( + "Optimized plan should contain LogicalDedup so DedupPushdownRule can match it:\n" + + optimizedExplain, + optimizedExplain.contains("LogicalDedup")); + Assert.assertFalse( + "Optimized plan should not retain ROW_NUMBER after simplification:\n" + optimizedExplain, + optimizedExplain.contains("ROW_NUMBER")); + Assert.assertTrue( + "User where predicate must be preserved as a filter below LogicalDedup:\n" + + optimizedExplain, + optimizedExplain.contains(">($5, 1000)")); + } + + /** + * Companion to {@link #testDedupAfterWhereProducesLogicalDedup} that pins the user-visible + * contract: with a {@code where} preceding {@code dedup}, a {@code LogicalDedup} must be produced + * regardless of the order in which {@link FilterMergeRule} and {@link PPLSimplifyDedupRule} fire. + * + *

The simplify rule's bucket-non-null operand predicate is order-independent — it accepts both + * a pure {@code IS NOT NULL} and an {@code AND} that contains an {@code IS NOT NULL} on a + * partition key — so {@code FilterMergeRule} firing first no longer disables dedup pushdown. This + * guards against re-introducing the original regression by reordering, removing, or adding rules + * to {@code CalciteToolsHelper#HEP_PROGRAM}. + */ + @Test + public void testDedupAfterWhereProducesLogicalDedupRegardlessOfRuleOrder() { + String ppl = "source=EMP | where SAL > 1000 | dedup DEPTNO"; + RelNode raw = getRelNodeRaw(ppl); + + HepProgram program = + new HepProgramBuilder() + .addRuleInstance(FilterMergeRule.Config.DEFAULT.toRule()) + .addRuleInstance(PPLSimplifyDedupRule.DEDUP_SIMPLIFY_RULE) + .build(); + RelNode optimized = runHepPlanner(raw, program); + + String optimizedExplain = optimized.explain(); + Assert.assertTrue( + "Even with FilterMergeRule firing first, PPLSimplifyDedupRule must still produce" + + " LogicalDedup so DedupPushdownRule can match it:\n" + + optimizedExplain, + optimizedExplain.contains("LogicalDedup")); + Assert.assertFalse( + "ROW_NUMBER window form should be removed after simplification:\n" + optimizedExplain, + optimizedExplain.contains("ROW_NUMBER")); + Assert.assertTrue( + "User where predicate must be preserved as a filter below LogicalDedup, even when" + + " FilterMergeRule fired first and folded it into the bucket-non-null filter:\n" + + optimizedExplain, + optimizedExplain.contains(">($5, 1000)")); + } + + /** + * Mirrors the exact shape of {@code CalciteToolsHelper.HEP_PROGRAM} used in production ({@code + * addRuleCollection(List.of(FilterMergeRule, PPLSimplifyDedupRule))}). The two sequenced-{@code + * addRuleInstance} tests above prove the rule fires under either ordering, but production runs + * both rules in a single collection instruction. This test pins that exact shape to catch any + * addRuleCollection-vs-addRuleInstance traversal differences. + */ + @Test + public void testDedupAfterWhereProducesLogicalDedupWithProductionHepProgram() { + String ppl = "source=EMP | where SAL > 1000 | dedup DEPTNO"; + RelNode raw = getRelNodeRaw(ppl); + + HepProgram program = + new HepProgramBuilder() + .addRuleCollection( + java.util.List.of( + FilterMergeRule.Config.DEFAULT.toRule(), + PPLSimplifyDedupRule.DEDUP_SIMPLIFY_RULE)) + .build(); + RelNode optimized = runHepPlanner(raw, program); + + String optimizedExplain = optimized.explain(); + Assert.assertTrue( + "Production HEP_PROGRAM (addRuleCollection) must still produce LogicalDedup:\n" + + optimizedExplain, + optimizedExplain.contains("LogicalDedup")); + Assert.assertFalse( + "ROW_NUMBER window form should be removed under production HEP_PROGRAM:\n" + + optimizedExplain, + optimizedExplain.contains("ROW_NUMBER")); + } + + private static RelNode runHepPlanner(RelNode root, HepProgram program) { + HepPlanner planner = new HepPlanner(program); + planner.setRoot(root); + return planner.findBestExp(); + } } From c6360e92ac57b80ecb6c30a112a6ce1d432dd406 Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Mon, 8 Jun 2026 12:48:57 -0700 Subject: [PATCH 13/36] Branch percentile and sum-null IT expectations for the analytics-engine route (#5522) CalcitePPLAggregationIT.testPercentile, testSumNull, and testSumGroupByNullValue hard-coded expectations from the Calcite DSL-pushdown path, so they failed when run through the analytics-engine (DataFusion) backend via -Dtests.analytics.parquet_indices=true: - percentile() is approximate. DataFusion's t-digest interpolation returns 46576 for percentile(balance, 90) where the OpenSearch/Calcite percentile_approx returns 48086 (p50 agrees). Both are valid approximations. - SUM over an all-null bucket is null per the SQL spec. The DSL-pushdown path returns 0 (a known quirk, #3408); DataFusion follows the spec like Calcite-no-pushdown and returns null. Branch the expected values on the existing isAnalyticsParquetIndicesEnabled() helper, matching the pattern already used in StatsCommandIT.testSumWithNull. No production code change; both engine paths now pass. Signed-off-by: Kai Huang --- .../remote/CalcitePPLAggregationIT.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index ec80e27ba5a..bd4c68b85de 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -980,7 +980,11 @@ public void testPercentile() throws IOException { "source=%s | stats percentile(balance, 50) as p50, percentile(balance, 90) as p90", TEST_INDEX_BANK)); verifySchema(actual, schema("p50", "bigint"), schema("p90", "bigint")); - verifyDataRows(actual, rows(32838, 48086)); + // percentile() is approximate. The analytics-engine backend (DataFusion) uses a different + // t-digest interpolation than the Calcite/OpenSearch percentile_approx implementation, so p90 + // lands on a different value (p50 agrees). Both are valid approximations. + int expectedP90 = isAnalyticsParquetIndicesEnabled() ? 46576 : 48086; + verifyDataRows(actual, rows(32838, expectedP90)); } @Test @@ -990,14 +994,18 @@ public void testSumGroupByNullValue() throws IOException { String.format( "source=%s | stats sum(balance) as a by age", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("a", null, "bigint"), schema("age", null, "int")); + // SUM of an all-null bucket is null per the SQL spec. The DSL-pushdown path returns 0 instead + // (a known pushdown quirk); the analytics-engine backend (DataFusion) follows the spec like + // Calcite-no-pushdown and returns null. See testSumNull and #3408. + Object emptySum = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows( response, - rows(isPushdownDisabled() ? null : 0, null), + rows(emptySum, null), rows(32838, 28), rows(39225, 32), rows(4180, 33), rows(48086, 34), - rows(isPushdownDisabled() ? null : 0, 36)); + rows(emptySum, 36)); } @Test @@ -1061,7 +1069,9 @@ public void testSumNull() throws IOException { + " ],\n" + " \"datarows\": [\n" + " [\n" - + (isPushdownDisabled() ? " null\n" : " 0\n") + + ((isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) + ? " null\n" + : " 0\n") + " ]\n" + " ],\n" + " \"total\": 1,\n" From 9a32644a181514288c13f529a2b61ea13cdce4da Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Mon, 8 Jun 2026 15:01:39 -0700 Subject: [PATCH 14/36] [BugFix] Key date_time pushdown on field type, not literal UDT (#5481) (#5515) Signed-off-by: Jialiang Liang --- .../sql/calcite/remote/CalcitePPLBasicIT.java | 19 +++ .../rest-api-spec/test/issues/5481.yml | 69 +++++++++ .../opensearch/request/PredicateAnalyzer.java | 73 +++++++--- .../request/PredicateAnalyzerTest.java | 132 ++++++++++++++++++ 4 files changed, 271 insertions(+), 22 deletions(-) create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index 27d971a4615..bf16324e604 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -515,6 +515,25 @@ public void testDateBetween() throws IOException { actual, rows("Nanette", "2018-06-23 00:00:00"), rows("Elinor", "2018-06-27 00:00:00")); } + /** + * A timestamp range comparison AND'd with an {@code IN} clause must push down and return rows. + */ + @Test + public void testTimestampRangeWithInClausePushDown() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where birthdate > timestamp('2018-06-01 00:00:00') | where state in" + + " ('IL', 'TN', 'WA') | fields firstname, state, birthdate", + TEST_INDEX_BANK)); + verifySchema( + actual, + schema("firstname", "string"), + schema("state", "string"), + schema("birthdate", "timestamp")); + verifyDataRows(actual, rows("Elinor", "WA", "2018-06-27 00:00:00")); + } + @Test public void testDateIn() throws IOException { // birthdate is a TIMESTAMP-typed field; the IN values are DATE literals. visitIn must compare diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml new file mode 100644 index 00000000000..4207a6c3005 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml @@ -0,0 +1,69 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5481 +# A timestamp range comparison AND'd with an IN clause on another field must push down and +# return rows. Calcite folds the IN into a Sarg and strips the timestamp literal's UDT; without +# the field-type-keyed fix the range query ships an unformatted date and the shard rejects it. +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + + - do: + indices.create: + index: issue5481 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + event_time: + type: date + severity: + type: keyword + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "issue5481", "_id": "1"}}' + - '{"event_time": "2026-05-28T10:00:00Z", "severity": "ERROR"}' + - '{"index": {"_index": "issue5481", "_id": "2"}}' + - '{"event_time": "2026-05-28T10:05:00Z", "severity": "WARN"}' + - '{"index": {"_index": "issue5481", "_id": "3"}}' + - '{"event_time": "2026-05-28T10:10:00Z", "severity": "INFO"}' + - '{"index": {"_index": "issue5481", "_id": "4"}}' + - '{"event_time": "2026-05-28T10:15:00Z", "severity": "ERROR"}' + - '{"index": {"_index": "issue5481", "_id": "5"}}' + - '{"event_time": "2026-05-28T10:20:00Z", "severity": "WARN"}' + - '{"index": {"_index": "issue5481", "_id": "6"}}' + - '{"event_time": "2026-05-28T10:25:00Z", "severity": "DEBUG"}' + +--- +teardown: + - do: + indices.delete: + index: issue5481 + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + +--- +"Issue 5481: timestamp range AND keyword IN pushes down and returns rows": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=issue5481 | where event_time > timestamp('2026-05-28 10:08:00') | where severity in ('ERROR', 'WARN') | fields severity, event_time | sort event_time + + - match: { total: 2 } + - match: { schema: [ { name: severity, type: "string" }, { name: event_time, type: "timestamp" } ] } + - match: { datarows: [ [ "ERROR", "2026-05-28 10:15:00" ], [ "WARN", "2026-05-28 10:20:00" ] ] } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 29f44adf086..67a31a7f56b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -1415,10 +1415,12 @@ public QueryExpression notLike(LiteralExpression literal) { @Override public QueryExpression equals(LiteralExpression literal) { - Object value = literal.value(); - if (literal.isDateTime()) { + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + if (isTimeStamp) { builder = - addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value).lte(value)); + addFormatIfNecessary( + isTimeStamp, rangeQuery(getFieldReference()).gte(value).lte(value)); } else { builder = termQuery(getFieldReferenceForTermQuery(), value); } @@ -1427,12 +1429,15 @@ public QueryExpression equals(LiteralExpression literal) { @Override public QueryExpression notEquals(LiteralExpression literal) { - Object value = literal.value(); - if (literal.isDateTime()) { + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + if (isTimeStamp) { builder = boolQuery() - .should(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gt(value))) - .should(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lt(value))); + .should( + addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).gt(value))) + .should( + addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).lt(value))); } else { builder = boolQuery() @@ -1445,32 +1450,48 @@ public QueryExpression notEquals(LiteralExpression literal) { @Override public QueryExpression gt(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gt(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).gt(value)); return this; } @Override public QueryExpression gte(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).gte(value)); return this; } @Override public QueryExpression lt(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lt(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).lt(value)); return this; } @Override public QueryExpression lte(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lte(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).lte(value)); return this; } + /** + * Whether the comparison is a timestamp/date range. The field type is the reliable signal: + * {@code literal.isDateTime()} reads the literal's UDT, which {@link + * org.apache.calcite.rex.RexSimplify} can strip (to VARCHAR) when a sibling clause is folded + * into a {@code Sarg}, e.g. {@code @timestamp > X AND severityText IN (...)}. Falling back to + * {@code rel.isTimeStampType()} keeps ISO-8601 normalization and the {@code "date_time"} format + * hint on the range query. + */ + private boolean isFieldOrLiteralDateTime(LiteralExpression literal) { + return literal.isDateTime() || (rel != null && rel.isTimeStampType()); + } + @Override public QueryExpression match(String query, Map optionalArguments) { builder = new MatchQuery().build(getFieldReference(), query, optionalArguments); @@ -1617,6 +1638,11 @@ public QueryExpression between(Range range, boolean isTimeStamp) { } private Object convertEndpointValue(Object value, boolean isTimeStamp) { + // Shared normalization entry point: guard a null endpoint so the timestamp branch's + // value.toString() cannot NPE. sargPointValue never produces null from a non-null input. + if (value == null) { + return null; + } value = sargPointValue(value); return isTimeStamp ? timestampValueForPushDown(value.toString()) : value; } @@ -1749,16 +1775,19 @@ public static ScriptSortBuilder.ScriptSortType getScriptSortType(RelDataType rel } /** - * By default, range queries on date/time need use the format of the source to parse the literal. - * So we need to specify that the literal has "date_time" format + * Range queries on date/time fields need the source format to parse the literal, so we attach the + * {@code "date_time"} format. The caller resolves whether the comparison is a timestamp range + * from the field type (see {@link SimpleQueryExpression#isFieldOrLiteralDateTime}) rather than + * the literal's UDT, which {@link org.apache.calcite.rex.RexSimplify} can strip when a sibling + * clause is folded into a {@code Sarg}. * - * @param literal literal value - * @param rangeQueryBuilder query builder to optionally add {@code format} expression - * @return existing builder with possible {@code format} attribute + * @param isTimeStamp whether the comparison endpoint is a timestamp/date range endpoint + * @param rangeQueryBuilder query builder to optionally add the {@code format} attribute + * @return the same builder, with {@code format("date_time")} added when {@code isTimeStamp} */ private static RangeQueryBuilder addFormatIfNecessary( - LiteralExpression literal, RangeQueryBuilder rangeQueryBuilder) { - if (literal.isDateTime()) { + boolean isTimeStamp, RangeQueryBuilder rangeQueryBuilder) { + if (isTimeStamp) { rangeQueryBuilder.format("date_time"); } return rangeQueryBuilder; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 3deb39d58c0..041063f62a0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -1147,6 +1147,138 @@ void notEquals_generatesBoolQueryForDateTime() throws ExpressionNotAnalyzableExc result.toString()); } + /** + * RexSimplify can strip the EXPR_TIMESTAMP UDT off a literal when a sibling clause is folded into + * a Sarg (e.g. {@code @timestamp > X AND severityText IN (...)}), leaving the literal as plain + * VARCHAR. The comparison must still emit a {@code format("date_time")} range query keyed off the + * field's type so the shard's default date parser accepts the value. + */ + @Test + void gt_normalizesVarcharLiteralAgainstTimestampField() throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.GREATER_THAN, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : false, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, + result.toString()); + } + + // Companion stripped-VARCHAR-literal tests for the remaining range shapes (equals -> gte+lte, + // notEquals -> two-should bool, lte -> single range). Each must produce the same DSL as its + // intact-UDT counterpart, proving the field-type fallback in isFieldOrLiteralDateTime keeps + // ISO-8601 normalization + format("date_time") on every comparison op, not just gt. See #5481. + @Test + void equals_normalizesVarcharLiteralAgainstTimestampField() + throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.EQUALS, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, + result.toString()); + } + + @Test + void notEquals_normalizesVarcharLiteralAgainstTimestampField() + throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.NOT_EQUALS, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(BoolQueryBuilder.class, result); + assertEquals( + """ + { + "bool" : { + "should" : [ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : false, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }, + { + "range" : { + "d" : { + "from" : null, + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : false, + "format" : "date_time", + "boost" : 1.0 + } + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, + result.toString()); + } + + @Test + void lte_normalizesVarcharLiteralAgainstTimestampField() throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : null, + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, + result.toString()); + } + @Test void gte_generatesRangeQueryWithFormatForDateTime() throws ExpressionNotAnalyzableException { RexNode call = From 74cdf9efbf6396f60ca6bea4a247eb7344ceff2e Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Mon, 8 Jun 2026 16:07:39 -0700 Subject: [PATCH 15/36] Fix CalcitePPLAggregationIT on the analytics-engine route (parquet testSimpleCount0 + APPROX_COUNT_DISTINCT name) (#5525) * Use a parquet-backed index in CalcitePPLAggregationIT.testSimpleCount0 A bare auto-created index isn't composite/parquet-backed, so on the analytics-engine route it doesn't route to the analytics engine. Switch to TEST_INDEX_BANK (loaded via loadIndex, which injects parquet settings when the flag is set, 7 docs) so the test is meaningful on both routes. Diagnosis by Sandesh Kumar. Signed-off-by: Kai Huang * Emit APPROX_COUNT_DISTINCT as the distinct_count_approx runtime name distinct_count_approx() failed to bind on the analytics-engine (DataFusion) route because the SqlAggFunction was named DISTINCT_COUNT_APPROX; the backend resolves aggregates by the Calcite/Substrait-standard name APPROX_COUNT_DISTINCT. The Java field name and PPL function name are unchanged. The OpenSearch V3 path is unaffected (it overrides this via the external HLL registration). Analytics-route binding is completed by opensearch-project/OpenSearch#22013. Per Sandesh Kumar. Signed-off-by: Kai Huang --------- Signed-off-by: Kai Huang --- .../expression/function/PPLBuiltinOperators.java | 3 ++- .../calcite/remote/CalcitePPLAggregationIT.java | 14 ++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 60172e70c84..a24015de992 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -521,7 +521,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlAggFunction DISTINCT_COUNT_APPROX = createUserDefinedAggFunction( DistinctCountApproxLogicalAggFunction.class, - "DISTINCT_COUNT_APPROX", + // Substrait-standard name the analytics-engine backend resolves by (V3 overrides it). + "APPROX_COUNT_DISTINCT", ReturnTypes.BIGINT_FORCE_NULLABLE, null); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index bd4c68b85de..5c8f6eb2cf7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -25,7 +25,6 @@ import java.util.List; import org.json.JSONObject; import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -52,16 +51,11 @@ public void init() throws Exception { @Test public void testSimpleCount0() throws IOException { - Request request1 = new Request("PUT", "/test/_doc/1?refresh=true"); - request1.setJsonEntity("{\"name\": \"hello\", \"age\": 20}"); - client().performRequest(request1); - Request request2 = new Request("PUT", "/test/_doc/2?refresh=true"); - request2.setJsonEntity("{\"name\": \"world\", \"age\": 30}"); - client().performRequest(request2); - - JSONObject actual = executeQuery("source=test | stats count() as c"); + // A bare auto-created index isn't parquet-backed; use the parquet-aware bank index (7 docs). + JSONObject actual = + executeQuery(String.format("source=%s | stats count() as c", TEST_INDEX_BANK)); verifySchema(actual, schema("c", "bigint")); - verifyDataRows(actual, rows(2)); + verifyDataRows(actual, rows(7)); } @Test From f2acb89e9150fe33f87734932530368329ac12f2 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Mon, 8 Jun 2026 16:08:23 -0700 Subject: [PATCH 16/36] [BugFix] Fix SQL aggregate window with ORDER BY defaulting to whole partition (#5526) Signed-off-by: Chen Dai --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 27 +++++++++++++++++++ .../sql/ast/expression/WindowBound.java | 2 ++ .../sql/ast/expression/WindowFrame.java | 7 +++++ .../sql/sql/parser/AstExpressionBuilder.java | 9 ++++++- .../sql/parser/AstExpressionBuilderTest.java | 17 ++++++++++-- 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index fb38f184925..2d214c8b5f4 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -424,6 +424,33 @@ GROUP BY department HAVING MAX(age) > 30 AND MIN(age) < 50 """); } + @Test + public void testCountDistinctWindowWithOrderBy() { + // No frame printed: RANGE .. CURRENT ROW is Calcite's default for ORDER BY. + givenQuery( + """ + SELECT department, COUNT(DISTINCT name) OVER(ORDER BY department) FROM catalog.employees + """) + .assertPlan( + """ + LogicalProject(department=[$3], COUNT(DISTINCT name) OVER(ORDER BY department)=[COUNT(DISTINCT $1) OVER (ORDER BY $3 NULLS FIRST)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testSumWindowWithPartitionAndOrderBy() { + givenQuery( + """ + SELECT name, SUM(age) OVER(PARTITION BY department ORDER BY age) FROM catalog.employees + """) + .assertPlan( + """ + LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + @Test public void testWindowOrderByDefaultsNullsFirst() { // Window function ORDER BY without explicit NULLS FIRST/LAST defaults to NULLS FIRST, diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java b/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java index d4241165cbc..62c6559a225 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ast.expression; +import lombok.EqualsAndHashCode; import lombok.Getter; public abstract class WindowBound { @@ -25,6 +26,7 @@ public boolean isPreceding() { } } + @EqualsAndHashCode(callSuper = false) public static class CurrentRowWindowBound extends WindowBound { CurrentRowWindowBound() {} diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java b/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java index 7ea1a072f8a..9e2fbf246d2 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java @@ -40,6 +40,13 @@ public static WindowFrame toCurrentRow() { AstDSL.stringLiteral("CURRENT ROW")); } + public static WindowFrame rangeToCurrentRow() { + return WindowFrame.of( + FrameType.RANGE, + AstDSL.stringLiteral("UNBOUNDED PRECEDING"), + AstDSL.stringLiteral("CURRENT ROW")); + } + public static WindowFrame of(FrameType type, String lower, String upper) { return WindowFrame.of(type, AstDSL.stringLiteral(lower), AstDSL.stringLiteral(upper)); } diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 273076af40f..33cfb1a56ca 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -222,7 +222,14 @@ public UnresolvedExpression visitWindowFunctionClause(WindowFunctionClauseContex .map(item -> ImmutablePair.of(createSortOption(item), visit(item.expression()))) .collect(Collectors.toList()); } - return new WindowFunction(visit(ctx.function), partitionByList, sortList); + UnresolvedExpression function = visit(ctx.function); + WindowFunction windowFunction = new WindowFunction(function, partitionByList, sortList); + + // Aggregate window with ORDER BY defaults to a running RANGE frame (ranking ignores it). + if (function instanceof AggregateFunction && !sortList.isEmpty()) { + windowFunction.setWindowFrame(WindowFrame.rangeToCurrentRow()); + } + return windowFunction; } @Override diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index e89f2af9b01..cb981f6f45f 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -44,6 +44,8 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ast.expression.WindowFrame; +import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; @@ -308,12 +310,23 @@ public void canBuildWindowFunctionWithoutOrderBy() { @Test public void canBuildAggregateWindowFunction() { + WindowFunction expected = + new WindowFunction( + aggregate("AVG", qualifiedName("age")), + ImmutableList.of(qualifiedName("state")), + ImmutableList.of(ImmutablePair.of(new SortOption(null, null), qualifiedName("age")))); + expected.setWindowFrame(WindowFrame.rangeToCurrentRow()); + assertEquals(expected, buildExprAst("AVG(age) OVER (PARTITION BY state ORDER BY age)")); + } + + @Test + public void canBuildAggregateWindowFunctionWithoutOrderBy() { assertEquals( window( aggregate("AVG", qualifiedName("age")), ImmutableList.of(qualifiedName("state")), - ImmutableList.of(ImmutablePair.of(new SortOption(null, null), qualifiedName("age")))), - buildExprAst("AVG(age) OVER (PARTITION BY state ORDER BY age)")); + ImmutableList.of()), + buildExprAst("AVG(age) OVER (PARTITION BY state)")); } @Test From f12e4c3910b87f833a01b3b05517907258c9dfd9 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Mon, 8 Jun 2026 16:08:35 -0700 Subject: [PATCH 17/36] [BugFix] Fix `FILTER(WHERE)` dropped on aggregates in unified SQL path (#5523) Signed-off-by: Chen Dai --- .../sql/api/UnifiedQueryPlannerSqlV2Test.java | 41 +++++++++++++++++++ .../sql/calcite/CalciteAggCallVisitor.java | 16 ++++++-- .../sql/calcite/CalciteRelNodeVisitor.java | 16 ++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index 2d214c8b5f4..db3d517f0b4 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -346,6 +346,47 @@ GROUP BY department HAVING MAX(age) > 30 """); } + @Test + public void testCountStarWithFilter() { + givenQuery("SELECT COUNT(*) FILTER(WHERE age > 30) FROM catalog.employees") + .assertPlan( + """ + LogicalAggregate(group=[{}], COUNT(*) FILTER(WHERE age > 30)=[COUNT() FILTER $0]) + LogicalProject($f1=[>($2, 30)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testFilteredAggregateWithGroupBy() { + givenQuery( + """ + SELECT department, SUM(age) FILTER(WHERE age > 30) FROM catalog.employees + GROUP BY department + """) + .assertPlan( + """ + LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[SUM($1) FILTER $2]) + LogicalProject(department=[$3], age=[$2], $f3=[>($2, 30)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testMultipleFilteredAggregates() { + givenQuery( + """ + SELECT MAX(age) FILTER(WHERE age > 30), MIN(age) FILTER(WHERE age < 50) + FROM catalog.employees + """) + .assertPlan( + """ + LogicalAggregate(group=[{}], MAX(age) FILTER(WHERE age > 30)=[MAX($0) FILTER $1], MIN(age) FILTER(WHERE age < 50)=[MIN($0) FILTER $2]) + LogicalProject(age=[$2], $f4=[>($2, 30)], $f5=[<($2, 50)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + @Test public void testScalarFnOverAggregate() { givenQuery("SELECT ABS(MAX(age)) FROM catalog.employees") diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java index 0512316628c..e7a5a5a68bd 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder.AggCall; import org.apache.logging.log4j.util.Strings; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -46,10 +47,17 @@ public AggCall visitAggregateFunction(AggregateFunction node, CalcitePlanContext } return BuiltinFunctionName.ofAggregation(node.getFuncName()) .map( - functionName -> { - return PlanUtils.makeAggCall( - context, functionName, node.getDistinct(), field, argList); - }) + functionName -> + PlanUtils.makeAggCall(context, functionName, node.getDistinct(), field, argList)) + // Apply the optional FILTER(WHERE ...) predicate; IS TRUE treats NULL as non-matching. + .map( + aggCall -> + node.condition() == null + ? aggCall + : aggCall.filter( + context.rexBuilder.makeCall( + SqlStdOperatorTable.IS_TRUE, + rexNodeVisitor.analyze(node.condition(), context)))) .orElseThrow( () -> new UnsupportedOperationException("Unexpected aggregation: " + node.getFuncName())); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 91a30361a20..ff2e9fbd4bd 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -1563,6 +1563,7 @@ private Pair, List> aggregateWithTrimming( List aggCallRefs = PlanUtils.getInputRefsFromAggCall(resolvedAggCallList); boolean hintNestedAgg = containsNestedAggregator(context.relBuilder, aggCallRefs); trimmedRefs.addAll(aggCallRefs); + trimmedRefs.addAll(getAggCallFilterRefs(aggExprList, context)); context.relBuilder.project(trimmedRefs); // Re-resolve all attributes based on adding trimmed Project. @@ -1795,6 +1796,21 @@ private static AggregateFunction extractAggregateFunction(UnresolvedExpression e return null; } + /** + * Collects input refs used by aggregate FILTER(WHERE ...) predicates so trimming retains them. + */ + private List getAggCallFilterRefs( + List aggExprList, CalcitePlanContext context) { + List refs = new ArrayList<>(); + for (UnresolvedExpression aggExpr : aggExprList) { + AggregateFunction aggFunc = extractAggregateFunction(aggExpr); + if (aggFunc != null && aggFunc.condition() != null) { + refs.addAll(PlanUtils.getInputRefs(rexVisitor.analyze(aggFunc.condition(), context))); + } + } + return refs; + } + private Optional getTimeSpanField(UnresolvedExpression expr) { if (Objects.isNull(expr)) return Optional.empty(); if (expr instanceof Span span && SpanUnit.isTimeUnit(span.getUnit())) { From f861d0275e07d5dd58b19e2a74eda26c21d17f9f Mon Sep 17 00:00:00 2001 From: Vinay Krishna Pudyodu Date: Mon, 8 Jun 2026 18:57:20 -0700 Subject: [PATCH 18/36] DATE/TIME label and rendering for date-only / time-only fields (#5521) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(core): cluster D — TIME-typed list elements format as HH:mm:ss Companion to OpenSearch fix/ai/datetime-clusters @ 9009736c2dc. list() now returns "HH:mm:ss[.fraction]" without the 1970-01-01 epoch-date prefix. The analytics-engine path rewrites PPL list() to DataFusion's list_merge, so the legacy ListAggFunction never fires. Instead, AnalyticsExecutionEngine now post-processes List-typed cells in the result conversion: when an element string matches "1970-01-01[ T]HH:mm:ss[.fraction]", only the time portion is kept. Scalar cells are untouched, preserving the wider timestamp-stringification regression baseline. Signed-off-by: Vinay Krishna Pudyodu * fix(core): cluster A — span() output column type for date/time UDT Recognize the new sandbox DateOnlyType / TimeOnlyType UDT markers in: - OpenSearchTypeFactory.convertAnalyticsEngineRelDataTypeToExprType: DateOnlyType → ExprCoreType.DATE, TimeOnlyType → ExprCoreType.TIME so the user-visible response schema labels span() bucket columns as `date` / `time` instead of `timestamp`. - AnalyticsExecutionEngine.toExprValue: when the column carries a DateOnlyType marker, strip the trailing ` HH:MM:SS` from the Timestamp(ms)-formatted wire value so dates render as `YYYY-MM-DD`. Symmetric handling for TimeOnlyType strips the `1970-01-01 ` prefix. Pairs with the sandbox schema-builder change in opensearch-project/OpenSearch@b69c5ff8888. Signed-off-by: Vinay Krishna Pudyodu * Preserve DATE/TIME return type for ADDDATE/SUBDATE/ADDTIME/SUBTIME on date/time columns On the analytics-engine route a 'date'/'time' column is a TIMESTAMP-backed DateOnlyType/TimeOnlyType marker, so operand-conditional return-type inference mis-read it as TIMESTAMP — ADDDATE(date_col, N) reported (and rendered) TIMESTAMP instead of DATE. Add isDateExprType/isTimeExprType helpers recognizing those markers (off the general conversion path, no substrait round-trip risk) and use them in AddSubDateFunction and PPLReturnTypes.TIME_APPLY_RETURN_TYPE. Fixes the 6 *Null column-operand cases end-to-end. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Vinay Krishna Pudyodu * Tighten ADDDATE/SUBDATE return-type helper comments Comment trims on top of d12407b1a (Marc Handalian's cherry-pick) — collapse the multi-line Javadoc on isDateExprType/isTimeExprType to one-line case names, and drop the inline narration at the two call sites (PPLReturnTypes.TIME_APPLY_RETURN_TYPE, AddSubDateFunction#getReturnTypeInference) that restated the helper contract. Behavior unchanged. Signed-off-by: Vinay Krishna Pudyodu * Fix tests Signed-off-by: Vinay Krishna Pudyodu --------- Signed-off-by: Vinay Krishna Pudyodu Co-authored-by: Claude Opus 4.8 (1M context) --- .../calcite/utils/OpenSearchTypeFactory.java | 19 ++++++ .../sql/calcite/utils/PPLReturnTypes.java | 4 +- .../analytics/AnalyticsExecutionEngine.java | 58 ++++++++++++----- .../udf/datetime/AddSubDateFunction.java | 2 +- .../AnalyticsExecutionEngineTest.java | 63 +++++++++++++++++++ 5 files changed, 127 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 9c15c1485c1..b8ed9b6cd7e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -47,7 +47,9 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.analytics.schema.BinaryType; +import org.opensearch.analytics.schema.DateOnlyType; import org.opensearch.analytics.schema.IpType; +import org.opensearch.analytics.schema.TimeOnlyType; import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.type.ExprBinaryType; import org.opensearch.sql.calcite.type.ExprDateType; @@ -277,6 +279,16 @@ public static ExprType convertRelDataTypeToExprType(RelDataType type) { return exprType; } + /** DATE check for return-type inference; recognizes the analytics-route {@link DateOnlyType}. */ + public static boolean isDateExprType(RelDataType type) { + return type instanceof DateOnlyType || convertRelDataTypeToExprType(type) == ExprCoreType.DATE; + } + + /** TIME counterpart of {@link #isDateExprType}; recognizes {@link TimeOnlyType}. */ + public static boolean isTimeExprType(RelDataType type) { + return type instanceof TimeOnlyType || convertRelDataTypeToExprType(type) == ExprCoreType.TIME; + } + /** * Result-schema-only variant of {@link #convertRelDataTypeToExprType} that recognizes the * analytics-engine {@link IpType} / {@link BinaryType} markers as {@link ExprCoreType#IP} / @@ -293,6 +305,13 @@ public static ExprType convertAnalyticsEngineRelDataTypeToExprType(RelDataType t if (type instanceof BinaryType) { return BINARY; } + // span() over date / time UDT — schema label is DATE / TIME, not TIMESTAMP. + if (type instanceof DateOnlyType) { + return DATE; + } + if (type instanceof TimeOnlyType) { + return TIME; + } return convertRelDataTypeToExprType(type); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java index acf2a6b8ebc..ce7dd408e7a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java @@ -12,7 +12,6 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.type.SqlTypeUtil; -import org.opensearch.sql.data.type.ExprCoreType; /** * Return types used in PPL. This class complements the {@link @@ -36,8 +35,7 @@ private PPLReturnTypes() {} public static SqlReturnTypeInference TIME_APPLY_RETURN_TYPE = opBinding -> { RelDataType temporalType = opBinding.getOperandType(0); - if (ExprCoreType.TIME.equals( - OpenSearchTypeFactory.convertRelDataTypeToExprType(temporalType))) { + if (OpenSearchTypeFactory.isTimeExprType(temporalType)) { return UserDefinedFunctionUtils.NULLABLE_TIME_UDT; } return UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT; diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java index 733c603a761..68dffacc2eb 100644 --- a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java @@ -12,6 +12,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; @@ -19,7 +20,9 @@ import org.opensearch.analytics.exec.QueryPlanExecutor; import org.opensearch.analytics.exec.profile.ProfiledResult; import org.opensearch.analytics.schema.BinaryType; +import org.opensearch.analytics.schema.DateOnlyType; import org.opensearch.analytics.schema.IpType; +import org.opensearch.analytics.schema.TimeOnlyType; import org.opensearch.common.network.InetAddresses; import org.opensearch.core.action.ActionListener; import org.opensearch.sql.ast.statement.ExplainMode; @@ -47,6 +50,15 @@ */ public class AnalyticsExecutionEngine implements ExecutionEngine { + // TIME-typed list elements round-trip via Timestamp and bypass ArrowValues' scalar + // post-processing (see DataFusion list_merge), arriving as "1970-01-01[ T]HH:mm:ss[.frac]". + private static final Pattern EPOCH_DATE_TIME_PREFIX = + Pattern.compile("^1970-01-01[ T](\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?)$"); + + // DateOnlyType wire is Timestamp(ms) at midnight — keep the date, drop the time. + private static final Pattern DATE_WITH_MIDNIGHT_TIME = + Pattern.compile("^(\\d{4}-\\d{2}-\\d{2})[ T]\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?$"); + private final QueryPlanExecutor> planExecutor; public AnalyticsExecutionEngine(QueryPlanExecutor> planExecutor) { @@ -206,21 +218,7 @@ private List convertRows(Iterable rows, List - *

  • {@link IpType} + {@code byte[]} → canonical address string (matches {@code - * IpFieldMapper}'s {@code valueFetcher} output). - *
  • {@link BinaryType} + {@code byte[]} → base64-encoded string (matches the OpenSearch - * {@code binary} field wire format). - *
  • Anything else → existing {@link ExprValueUtils#fromObjectValue} path. - * - * - *

    Without this dispatch, {@code fromObjectValue} throws {@code unsupported object class [B} on - * byte[] cells, and IP buffers leak through as raw 16-byte ipv4-mapped-ipv6 garbage. - */ + /** Renders UDT cells (IP/binary byte[]; date / time string) and strips TIME prefixes in lists. */ private static ExprValue toExprValue(Object value, RelDataType type) { if (value instanceof byte[] bytes) { if (type instanceof IpType) { @@ -234,9 +232,39 @@ private static ExprValue toExprValue(Object value, RelDataType type) { return ExprValueUtils.stringValue(Base64.getEncoder().encodeToString(bytes)); } } + // DateOnlyType scalar — strip midnight time suffix off the Timestamp(ms) wire. + if (type instanceof DateOnlyType && value instanceof String s) { + var m = DATE_WITH_MIDNIGHT_TIME.matcher(s); + if (m.matches()) { + return ExprValueUtils.stringValue(m.group(1)); + } + } + // TimeOnlyType scalar — strip 1970-01-01 prefix off the Timestamp(ms) wire. + if (type instanceof TimeOnlyType && value instanceof String s) { + var m = EPOCH_DATE_TIME_PREFIX.matcher(s); + if (m.matches()) { + return ExprValueUtils.stringValue(m.group(1)); + } + } + if (value instanceof List list) { + return ExprValueUtils.collectionValue(stripEpochDatePrefixInList(list)); + } return ExprValueUtils.fromObjectValue(value); } + private static List stripEpochDatePrefixInList(List list) { + List out = new ArrayList<>(list.size()); + for (Object element : list) { + if (element instanceof String s) { + var m = EPOCH_DATE_TIME_PREFIX.matcher(s); + out.add(m.matches() ? m.group(1) : s); + } else { + out.add(element); + } + } + return out; + } + private Schema buildSchema(List fields) { List columns = new ArrayList<>(); for (RelDataTypeField field : fields) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java index 9456e6b857a..2c040918e80 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java @@ -63,7 +63,7 @@ public SqlReturnTypeInference getReturnTypeInference() { return opBinding -> { RelDataType temporalType = opBinding.getOperandType(0); RelDataType temporalDeltaType = opBinding.getOperandType(1); - if (OpenSearchTypeFactory.convertRelDataTypeToExprType(temporalType) == ExprCoreType.DATE + if (OpenSearchTypeFactory.isDateExprType(temporalType) && SqlTypeFamily.NUMERIC.contains(temporalDeltaType)) { return NULLABLE_DATE_UDT; } else { diff --git a/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java b/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java index e759b128a45..08b88048468 100644 --- a/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java @@ -30,7 +30,9 @@ import org.junit.jupiter.api.Test; import org.opensearch.analytics.exec.QueryPlanExecutor; import org.opensearch.analytics.schema.BinaryType; +import org.opensearch.analytics.schema.DateOnlyType; import org.opensearch.analytics.schema.IpType; +import org.opensearch.analytics.schema.TimeOnlyType; import org.opensearch.core.action.ActionListener; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.SysLimit; @@ -237,6 +239,67 @@ void executeRelNode_binaryColumnRendersAsBase64() { "byte[] should base64-encode to match OpenSearch binary wire format. " + dump); } + /** DateOnlyType — schema reports DATE, value strips midnight suffix. */ + @Test + void executeRelNode_dateOnlyTypeStripsTimeSuffix() { + RelNode relNode = + mockRelNodeWithType("d", new DateOnlyType(RelDataTypeSystem.DEFAULT, true, 3)); + Iterable rows = Collections.singletonList(new Object[] {"1984-04-12 00:00:00"}); + stubExecutorWith(relNode, rows); + + QueryResponse response = executeAndCapture(relNode); + String dump = dumpResponse(response); + + assertEquals(ExprCoreType.DATE, response.getSchema().getColumns().get(0).getExprType(), dump); + assertEquals( + "1984-04-12", response.getResults().get(0).tupleValue().get("d").stringValue(), dump); + } + + /** TimeOnlyType — schema reports TIME, value strips 1970-01-01 prefix. */ + @Test + void executeRelNode_timeOnlyTypeStripsEpochDatePrefix() { + RelNode relNode = + mockRelNodeWithType("t", new TimeOnlyType(RelDataTypeSystem.DEFAULT, true, 3)); + Iterable rows = Collections.singletonList(new Object[] {"1970-01-01 09:00:00"}); + stubExecutorWith(relNode, rows); + + QueryResponse response = executeAndCapture(relNode); + String dump = dumpResponse(response); + + assertEquals(ExprCoreType.TIME, response.getSchema().getColumns().get(0).getExprType(), dump); + assertEquals( + "09:00:00", response.getResults().get(0).tupleValue().get("t").stringValue(), dump); + } + + /** TIME-typed list elements arrive as "1970-01-01[ T]HH:mm:ss[.frac]" — strip the prefix. */ + @Test + void executeRelNode_listOfStringStripsEpochDatePrefix() { + SqlTypeFactoryImpl typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + RelDataType arrayOfVarchar = + typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.VARCHAR), -1); + RelNode relNode = mockRelNodeWithType("time_list", arrayOfVarchar); + java.util.List input = + Arrays.asList( + "1970-01-01 19:36:22", + "1970-01-01T02:05:25", + "1970-01-01 12:34:56.123456789", + "2020-10-13 13:00:00", + "hello"); + stubExecutorWith(relNode, Collections.singletonList(new Object[] {input})); + + QueryResponse response = executeAndCapture(relNode); + String dump = dumpResponse(response); + + java.util.List result = + response.getResults().get(0).tupleValue().get("time_list").collectionValue().stream() + .map(org.opensearch.sql.data.model.ExprValue::stringValue) + .toList(); + assertEquals( + Arrays.asList("19:36:22", "02:05:25", "12:34:56.123456789", "2020-10-13 13:00:00", "hello"), + result, + dump); + } + @Test void executeRelNode_emptyResults() { RelNode relNode = mockRelNode("name", SqlTypeName.VARCHAR); From ab58c4d25534bb676caffce1348534be4a715b3d Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 9 Jun 2026 08:16:06 -0700 Subject: [PATCH 19/36] Exclude full-text search-filter operator tests on the analytics-engine route (#5527) The 7 comparison-operator tests in OperatorIT / CalciteOperatorIT use the search-filter syntax (source=idx age = 32), which lowers to a Lucene query_string predicate. Executing it on the analytics-engine route fails with a null LuceneReader: query_string needs an inverted-index searcher, which parquet-backed analytics indices don't have (DataFusion is not a full-text engine). Full-text search is genuinely unsupported there. Exclude these tests only when the analytics route is actually enabled (Boolean.parseBoolean(tests.analytics.parquet_indices), matching AnalyticsIndexConfig.isEnabled) so the v2 path still runs them, following the established build.gradle exclusion pattern. Signed-off-by: Kai Huang --- integ-test/build.gradle | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 21741bffff3..6554b7c8491 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1035,13 +1035,30 @@ task integTestRemote(type: RestIntegTestTask) { systemProperty 'tests.analytics.parquet_indices', System.getProperty("tests.analytics.parquet_indices") } + // True only when the analytics-engine route is active (every test index parquet-backed). Matches + // AnalyticsIndexConfig.isEnabled, which parses the value rather than checking mere presence, so a + // `-Dtests.analytics.parquet_indices=false` run stays on the v2 path. + def analyticsEnabled = Boolean.parseBoolean(System.getProperty("tests.analytics.parquet_indices", "false")) + // Set default query size limit systemProperty 'defaultQuerySizeLimit', '10000' - if (System.getProperty("tests.rest.bwcsuite") == null) { - filter { + filter { + if (System.getProperty("tests.rest.bwcsuite") == null) { excludeTestsMatching "org.opensearch.sql.bwc.*IT" } + + if (analyticsEnabled) { + // Full-text search (search-filter syntax -> Lucene query_string) needs an inverted-index + // reader that parquet-backed analytics indices lack, so it's unsupported on this route. + excludeTestsMatching '*OperatorIT.testEqualOperator' + excludeTestsMatching '*OperatorIT.testNotEqualOperator' + excludeTestsMatching '*OperatorIT.testLessOperator' + excludeTestsMatching '*OperatorIT.testLteOperator' + excludeTestsMatching '*OperatorIT.testGreaterOperator' + excludeTestsMatching '*OperatorIT.testGteOperator' + excludeTestsMatching '*OperatorIT.testNotOperator' + } } // Exclude the same tests that are excluded for integTest From 0bff6096e2c7c2faa59a45013e728644dbd8611b Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Tue, 9 Jun 2026 13:10:47 -0700 Subject: [PATCH 20/36] Parquet-back raw-PUT test indices on the analytics-engine route (#5529) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With -Dtests.analytics.parquet_indices=true, indices created by a raw document PUT (e.g. `PUT /test/_doc/1` in a test's init()) bypass AnalyticsIndexConfig.applyIndexCreationSettings, so they inherit the composite *value* — and are therefore routed to the analytics engine by RestUnifiedQueryAction.isAnalyticsIndex — but not the `pluggable.dataformat.enabled` flag. They are then stored as a plain-Lucene EngineBackedIndexer whose acquireReader() is unimplemented, and the query fails with `StreamException[INTERNAL] Failed to start streaming fragment`. Apply the cluster-level composite defaults in setUpIndices() so every index — including raw-PUT ones — is stored as a parquet-backed DataFormatAwareEngine that is actually scannable by the analytics engine it routes to. No-op unless tests.analytics.parquet_indices=true, so normal CI is unchanged. Signed-off-by: Kai Huang --- .../sql/legacy/SQLIntegTestCase.java | 6 +++ .../org/opensearch/sql/legacy/TestUtils.java | 37 +++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 099b2f7e0cb..34bb5ae1252 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -66,6 +66,12 @@ public void setUpIndices() throws Exception { initClient(); } + // When -Dtests.analytics.parquet_indices=true, make every index (including ones a test + // auto-creates via a raw document PUT, which bypasses createIndexByRestClient) parquet-backed + // composite, so it is stored as a DataFormatAwareEngine and is actually scannable by the + // analytics engine it routes to. Must run before init() creates any index. + TestUtils.AnalyticsIndexConfig.applyClusterSettings(client()); + if (shouldResetQuerySizeLimit()) { resetQuerySizeLimit(); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 681c28c0a05..5e63613edb0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Locale; import java.util.stream.Collectors; +import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -68,6 +69,11 @@ public static boolean isEnabled() { return Boolean.parseBoolean(System.getProperty(ENABLED_PROP, "false")); } + // Composite-store format values shared by the index-level and cluster-level settings below. + private static final String DATAFORMAT_COMPOSITE = "composite"; + private static final String PRIMARY_FORMAT_PARQUET = "parquet"; + private static final String SECONDARY_FORMAT_LUCENE = "lucene"; + /** * Inject the parquet-backed composite-store index settings into {@code jsonObject}. No-op when * the config is disabled; idempotent — safe on any index-creation JSON shape. @@ -82,13 +88,38 @@ static void applyIndexCreationSettings(JSONObject jsonObject) { settings.has("index") ? settings.getJSONObject("index") : new JSONObject(); indexSettings.put("number_of_shards", 1); indexSettings.put("pluggable.dataformat.enabled", true); - indexSettings.put("pluggable.dataformat", "composite"); - indexSettings.put("composite.primary_data_format", "parquet"); - indexSettings.put("composite.secondary_data_formats", new org.json.JSONArray().put("lucene")); + indexSettings.put("pluggable.dataformat", DATAFORMAT_COMPOSITE); + indexSettings.put("composite.primary_data_format", PRIMARY_FORMAT_PARQUET); + indexSettings.put( + "composite.secondary_data_formats", new JSONArray().put(SECONDARY_FORMAT_LUCENE)); settings.put("index", indexSettings); jsonObject.put("settings", settings); } + /** + * Set the composite-store defaults at the cluster level so even indices auto-created by a raw + * document {@code PUT} (which bypass {@link #applyIndexCreationSettings}) are parquet-backed. + * Otherwise such an index inherits only the composite value — so it routes to the analytics + * engine — but not the {@code .enabled} flag, leaving it stored as a plain-Lucene {@code + * EngineBackedIndexer} that fails at query time. No-op when disabled; idempotent. + */ + public static void applyClusterSettings(RestClient client) { + if (!isEnabled()) { + return; + } + JSONObject persistent = + new JSONObject() + .put("cluster.pluggable.dataformat.enabled", true) + .put("cluster.pluggable.dataformat", DATAFORMAT_COMPOSITE) + .put("cluster.composite.primary_data_format", PRIMARY_FORMAT_PARQUET) + .put( + "cluster.composite.secondary_data_formats", + new JSONArray().put(SECONDARY_FORMAT_LUCENE)); + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity(new JSONObject().put("persistent", persistent).toString()); + performRequest(client, request); + } + /** * Returns the {@code _bulk} refresh query string for the current index type. Parquet-backed * indices in the analytics-backend-lucene composite engine don't yet implement {@code From 785e2777e52fb945ef5d3b6a57a59d0805b50c59 Mon Sep 17 00:00:00 2001 From: Radhakrishnan Pachyappan Date: Wed, 10 Jun 2026 02:12:13 +0530 Subject: [PATCH 21/36] feat(test): add WHERE-prefix view infrastructure for cross-command testing (#5505) (#5508) Signed-off-by: Radhakrishnan Pachyappan --- .../sql/legacy/SQLIntegTestCase.java | 10 + .../org/opensearch/sql/legacy/TestUtils.java | 10 + .../opensearch/sql/legacy/TestsConstants.java | 2 + .../opensearch/sql/ppl/FieldsCommandIT.java | 44 +- .../opensearch/sql/ppl/PPLIntegTestCase.java | 35 + .../src/test/resources/accounts_extended.json | 2006 +++++++++++++++++ .../src/test/resources/bank_extended.json | 20 + .../account_extended_index_mapping.json | 53 + .../bank_extended_index_mapping.json | 61 + 9 files changed, 2226 insertions(+), 15 deletions(-) create mode 100644 integ-test/src/test/resources/accounts_extended.json create mode 100644 integ-test/src/test/resources/bank_extended.json create mode 100644 integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json create mode 100644 integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 34bb5ae1252..bde077c20c3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -539,6 +539,11 @@ public enum Index { "account", getAccountIndexMapping(), "src/test/resources/accounts.json"), + ACCOUNT_EXTENDED( + TestsConstants.TEST_INDEX_ACCOUNT_EXTENDED, + "account_extended", + getAccountExtendedIndexMapping(), + "src/test/resources/accounts_extended.json"), PHRASE( TestsConstants.TEST_INDEX_PHRASE, "phrase", @@ -632,6 +637,11 @@ public enum Index { "account", getBankIndexMapping(), "src/test/resources/bank.json"), + BANK_EXTENDED( + TestsConstants.TEST_INDEX_BANK_EXTENDED, + "bank_extended", + getBankExtendedIndexMapping(), + "src/test/resources/bank_extended.json"), BANK_TWO( TestsConstants.TEST_INDEX_BANK_TWO, "account_two", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 5e63613edb0..e3af81fc44a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -262,6 +262,11 @@ public static String getAccountIndexMapping() { return getMappingFile(mappingFile); } + public static String getAccountExtendedIndexMapping() { + String mappingFile = "account_extended_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getPhraseIndexMapping() { String mappingFile = "phrase_index_mapping.json"; return getMappingFile(mappingFile); @@ -334,6 +339,11 @@ public static String getBankIndexMapping() { return getMappingFile(mappingFile); } + public static String getBankExtendedIndexMapping() { + String mappingFile = "bank_extended_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getGeoIpIndexMapping() { String mappingFile = "geoip_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 574f7b1eedb..f78deb532c4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -15,6 +15,7 @@ public class TestsConstants { public static final String TEST_INDEX_ONLINE = TEST_INDEX + "_online"; public static final String TEST_INDEX_ACCOUNT = TEST_INDEX + "_account"; + public static final String TEST_INDEX_ACCOUNT_EXTENDED = TEST_INDEX_ACCOUNT + "_extended"; public static final String TEST_INDEX_PHRASE = TEST_INDEX + "_phrase"; public static final String TEST_INDEX_DOG = TEST_INDEX + "_dog"; public static final String TEST_INDEX_DOG2 = TEST_INDEX + "_dog2"; @@ -39,6 +40,7 @@ public class TestsConstants { public static final String TEST_INDEX_JOIN_TYPE = TEST_INDEX + "_join_type"; public static final String TEST_INDEX_UNEXPANDED_OBJECT = TEST_INDEX + "_unexpanded_object"; public static final String TEST_INDEX_BANK = TEST_INDEX + "_bank"; + public static final String TEST_INDEX_BANK_EXTENDED = TEST_INDEX_BANK + "_extended"; public static final String TEST_INDEX_BANK_TWO = TEST_INDEX_BANK + "_two"; public static final String TEST_INDEX_BANK_WITH_NULL_VALUES = TEST_INDEX_BANK + "_with_null_values"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index 3f078fe6512..4d755c1ab77 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -16,9 +16,12 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.stream.Stream; import org.json.JSONObject; import org.junit.Ignore; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; public class FieldsCommandIT extends PPLIntegTestCase { @@ -26,25 +29,37 @@ public class FieldsCommandIT extends PPLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.ACCOUNT); + loadIndex(Index.ACCOUNT_EXTENDED); loadIndex(Index.BANK); + loadIndex(Index.BANK_EXTENDED); loadIndex(Index.MERGE_TEST_1); loadIndex(Index.MERGE_TEST_2); } - @Test - public void testBasicFieldSelection() throws IOException { - JSONObject result = - executeQuery(String.format("source=%s | fields firstname, lastname", TEST_INDEX_ACCOUNT)); + // --- Parameterized sources --- + + static Stream accountIndexSources() { + return sourceViews(TEST_INDEX_ACCOUNT); + } + + static Stream bankIndexSources() { + return sourceViews(TEST_INDEX_BANK); + } + + // --- Tests --- + + @ParameterizedTest(name = "querySource={0}") + @MethodSource("accountIndexSources") + public void testBasicFieldSelection(String querySource) throws IOException { + JSONObject result = executeQuery(querySource + " | fields firstname, lastname"); verifyColumn(result, columnName("firstname"), columnName("lastname")); verifySchema(result, schema("firstname", "string"), schema("lastname", "string")); } - @Test - public void testMultipleFieldSelection() throws IOException { - JSONObject result = - executeQuery( - String.format( - "source=%s | fields firstname, lastname, age | head 3", TEST_INDEX_ACCOUNT)); + @ParameterizedTest(name = "querySource={0}") + @MethodSource("accountIndexSources") + public void testMultipleFieldSelection(String querySource) throws IOException { + JSONObject result = executeQuery(querySource + " | fields firstname, lastname, age | head 3"); verifySchema( result, schema("firstname", "string"), @@ -57,12 +72,11 @@ public void testMultipleFieldSelection() throws IOException { rows("Nanette", "Bates", 28)); } - @Test - public void testSpecialDataTypes() throws IOException { - JSONObject result = - executeQuery(String.format("source=%s | fields birthdate", TEST_INDEX_BANK)); + @ParameterizedTest(name = "querySource={0}") + @MethodSource("bankIndexSources") + public void testSpecialDataTypes(String querySource) throws IOException { + JSONObject result = executeQuery(querySource + " | fields birthdate"); verifySchema(result, schema("birthdate", null, "timestamp")); - verifyDataRows( result, rows("2017-10-23 00:00:00"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 6c25c415af1..3db2142cffe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -6,6 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT_EXTENDED; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_EXTENDED; import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; import com.google.common.io.Resources; @@ -14,6 +18,8 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Locale; +import java.util.Map; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONException; @@ -148,6 +154,35 @@ protected static String source(String index, String query) { return String.format("source=%s | %s", index, query); } + // Maps each base index to its extended counterpart with synthetic rows. + private static final Map INDEX_VIEWS = + Map.of( + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT_EXTENDED, + TEST_INDEX_BANK, TEST_INDEX_BANK_EXTENDED); + + /** + * Returns a WHERE-prefix view source string over an extended index. The extended index is + * expected to contain all rows from the base index with {@code "_is_real":true}, plus additional + * synthetic rows with {@code "_is_real":false}. Use this as a drop-in replacement for {@code + * "source="} in parameterized tests to verify commands work correctly when preceded by a + * {@code where} filter. + */ + protected static String sourceView(String extendedIndex) { + return String.format("source=%s | where _is_real | fields - _is_real", extendedIndex); + } + + /** + * Returns a stream of parameterized source strings for the given base index: a direct source and, + * if an extended view is registered in {@link #INDEX_VIEWS}, a WHERE-prefix view over it. New + * views can be added by updating {@link #INDEX_VIEWS} without changing individual test files. + */ + protected static Stream sourceViews(String baseIndex) { + String extendedIndex = INDEX_VIEWS.get(baseIndex); + return extendedIndex != null + ? Stream.of(String.format("source=%s", baseIndex), sourceView(extendedIndex)) + : Stream.of(String.format("source=%s", baseIndex)); + } + protected void timing(MapBuilder builder, String query, String ppl) throws IOException { executeQuery(ppl); // warm-up diff --git a/integ-test/src/test/resources/accounts_extended.json b/integ-test/src/test/resources/accounts_extended.json new file mode 100644 index 00000000000..9af355d797c --- /dev/null +++ b/integ-test/src/test/resources/accounts_extended.json @@ -0,0 +1,2006 @@ +{"index":{"_id":"1"}} +{"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL","_is_real":true} +{"index":{"_id":"6"}} +{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN","_is_real":true} +{"index":{"_id":"13"}} +{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA","_is_real":true} +{"index":{"_id":"18"}} +{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":"Boink","email":"daleadams@boink.com","city":"Orick","state":"MD","_is_real":true} +{"index":{"_id":"20"}} +{"account_number":20,"balance":16418,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place","employer":"Scentric","email":"elinorratliff@scentric.com","city":"Ribera","state":"WA","_is_real":true} +{"index":{"_id":"25"}} +{"account_number":25,"balance":40540,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue","employer":"Filodyne","email":"virginiaayala@filodyne.com","city":"Nicholson","state":"PA","_is_real":true} +{"index":{"_id":"32"}} +{"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street","employer":"Quailcom","email":"dillardmcpherson@quailcom.com","city":"Veguita","state":"IN","_is_real":true} +{"index":{"_id":"37"}} +{"account_number":37,"balance":18612,"firstname":"Mcgee","lastname":"Mooney","age":39,"gender":"M","address":"826 Fillmore Place","employer":"Reversus","email":"mcgeemooney@reversus.com","city":"Tooleville","state":"OK","_is_real":true} +{"index":{"_id":"44"}} +{"account_number":44,"balance":34487,"firstname":"Aurelia","lastname":"Harding","age":37,"gender":"M","address":"502 Baycliff Terrace","employer":"Orbalix","email":"aureliaharding@orbalix.com","city":"Yardville","state":"DE","_is_real":true} +{"index":{"_id":"49"}} +{"account_number":49,"balance":29104,"firstname":"Fulton","lastname":"Holt","age":23,"gender":"F","address":"451 Humboldt Street","employer":"Anocha","email":"fultonholt@anocha.com","city":"Sunriver","state":"RI","_is_real":true} +{"index":{"_id":"51"}} +{"account_number":51,"balance":14097,"firstname":"Burton","lastname":"Meyers","age":31,"gender":"F","address":"334 River Street","employer":"Bezal","email":"burtonmeyers@bezal.com","city":"Jacksonburg","state":"MO","_is_real":true} +{"index":{"_id":"56"}} +{"account_number":56,"balance":14992,"firstname":"Josie","lastname":"Nelson","age":32,"gender":"M","address":"857 Tabor Court","employer":"Emtrac","email":"josienelson@emtrac.com","city":"Sunnyside","state":"UT","_is_real":true} +{"index":{"_id":"63"}} +{"account_number":63,"balance":6077,"firstname":"Hughes","lastname":"Owens","age":30,"gender":"F","address":"510 Sedgwick Street","employer":"Valpreal","email":"hughesowens@valpreal.com","city":"Guilford","state":"KS","_is_real":true} +{"index":{"_id":"68"}} +{"account_number":68,"balance":44214,"firstname":"Hall","lastname":"Key","age":25,"gender":"F","address":"927 Bay Parkway","employer":"Eventex","email":"hallkey@eventex.com","city":"Shawmut","state":"CA","_is_real":true} +{"index":{"_id":"70"}} +{"account_number":70,"balance":38172,"firstname":"Deidre","lastname":"Thompson","age":33,"gender":"F","address":"685 School Lane","employer":"Netplode","email":"deidrethompson@netplode.com","city":"Chestnut","state":"GA","_is_real":true} +{"index":{"_id":"75"}} +{"account_number":75,"balance":40500,"firstname":"Sandoval","lastname":"Kramer","age":22,"gender":"F","address":"166 Irvington Place","employer":"Overfork","email":"sandovalkramer@overfork.com","city":"Limestone","state":"NH","_is_real":true} +{"index":{"_id":"82"}} +{"account_number":82,"balance":41412,"firstname":"Concetta","lastname":"Barnes","age":39,"gender":"F","address":"195 Bayview Place","employer":"Fitcore","email":"concettabarnes@fitcore.com","city":"Summerfield","state":"NC","_is_real":true} +{"index":{"_id":"87"}} +{"account_number":87,"balance":1133,"firstname":"Hewitt","lastname":"Kidd","age":22,"gender":"M","address":"446 Halleck Street","employer":"Isologics","email":"hewittkidd@isologics.com","city":"Coalmont","state":"ME","_is_real":true} +{"index":{"_id":"94"}} +{"account_number":94,"balance":41060,"firstname":"Brittany","lastname":"Cabrera","age":30,"gender":"F","address":"183 Kathleen Court","employer":"Mixers","email":"brittanycabrera@mixers.com","city":"Cornucopia","state":"AZ","_is_real":true} +{"index":{"_id":"99"}} +{"account_number":99,"balance":47159,"firstname":"Ratliff","lastname":"Heath","age":39,"gender":"F","address":"806 Rockwell Place","employer":"Zappix","email":"ratliffheath@zappix.com","city":"Shaft","state":"ND","_is_real":true} +{"index":{"_id":"102"}} +{"account_number":102,"balance":29712,"firstname":"Dena","lastname":"Olson","age":27,"gender":"F","address":"759 Newkirk Avenue","employer":"Hinway","email":"denaolson@hinway.com","city":"Choctaw","state":"NJ","_is_real":true} +{"index":{"_id":"107"}} +{"account_number":107,"balance":48844,"firstname":"Randi","lastname":"Rich","age":28,"gender":"M","address":"694 Jefferson Street","employer":"Netplax","email":"randirich@netplax.com","city":"Bellfountain","state":"SC","_is_real":true} +{"index":{"_id":"114"}} +{"account_number":114,"balance":43045,"firstname":"Josephine","lastname":"Joseph","age":31,"gender":"F","address":"451 Oriental Court","employer":"Turnabout","email":"josephinejoseph@turnabout.com","city":"Sedley","state":"AL","_is_real":true} +{"index":{"_id":"119"}} +{"account_number":119,"balance":49222,"firstname":"Laverne","lastname":"Johnson","age":28,"gender":"F","address":"302 Howard Place","employer":"Senmei","email":"lavernejohnson@senmei.com","city":"Herlong","state":"DC","_is_real":true} +{"index":{"_id":"121"}} +{"account_number":121,"balance":19594,"firstname":"Acevedo","lastname":"Dorsey","age":32,"gender":"M","address":"479 Nova Court","employer":"Netropic","email":"acevedodorsey@netropic.com","city":"Islandia","state":"CT","_is_real":true} +{"index":{"_id":"126"}} +{"account_number":126,"balance":3607,"firstname":"Effie","lastname":"Gates","age":39,"gender":"F","address":"620 National Drive","employer":"Digitalus","email":"effiegates@digitalus.com","city":"Blodgett","state":"MD","_is_real":true} +{"index":{"_id":"133"}} +{"account_number":133,"balance":26135,"firstname":"Deena","lastname":"Richmond","age":36,"gender":"F","address":"646 Underhill Avenue","employer":"Sunclipse","email":"deenarichmond@sunclipse.com","city":"Austinburg","state":"SC","_is_real":true} +{"index":{"_id":"138"}} +{"account_number":138,"balance":9006,"firstname":"Daniel","lastname":"Arnold","age":39,"gender":"F","address":"422 Malbone Street","employer":"Ecstasia","email":"danielarnold@ecstasia.com","city":"Gardiner","state":"MO","_is_real":true} +{"index":{"_id":"140"}} +{"account_number":140,"balance":26696,"firstname":"Cotton","lastname":"Christensen","age":32,"gender":"M","address":"878 Schermerhorn Street","employer":"Prowaste","email":"cottonchristensen@prowaste.com","city":"Mayfair","state":"LA","_is_real":true} +{"index":{"_id":"145"}} +{"account_number":145,"balance":47406,"firstname":"Rowena","lastname":"Wilkinson","age":32,"gender":"M","address":"891 Elton Street","employer":"Asimiline","email":"rowenawilkinson@asimiline.com","city":"Ripley","state":"NH","_is_real":true} +{"index":{"_id":"152"}} +{"account_number":152,"balance":8088,"firstname":"Wolfe","lastname":"Rocha","age":21,"gender":"M","address":"457 Guernsey Street","employer":"Hivedom","email":"wolferocha@hivedom.com","city":"Adelino","state":"MS","_is_real":true} +{"index":{"_id":"157"}} +{"account_number":157,"balance":39868,"firstname":"Claudia","lastname":"Terry","age":20,"gender":"F","address":"132 Gunnison Court","employer":"Lumbrex","email":"claudiaterry@lumbrex.com","city":"Castleton","state":"MD","_is_real":true} +{"index":{"_id":"164"}} +{"account_number":164,"balance":9101,"firstname":"Cummings","lastname":"Little","age":26,"gender":"F","address":"308 Schaefer Street","employer":"Comtrak","email":"cummingslittle@comtrak.com","city":"Chaparrito","state":"WI","_is_real":true} +{"index":{"_id":"169"}} +{"account_number":169,"balance":45953,"firstname":"Hollie","lastname":"Osborn","age":34,"gender":"M","address":"671 Seaview Court","employer":"Musaphics","email":"hollieosborn@musaphics.com","city":"Hanover","state":"GA","_is_real":true} +{"index":{"_id":"171"}} +{"account_number":171,"balance":7091,"firstname":"Nelda","lastname":"Hopper","age":39,"gender":"M","address":"742 Prospect Place","employer":"Equicom","email":"neldahopper@equicom.com","city":"Finderne","state":"SC","_is_real":true} +{"index":{"_id":"176"}} +{"account_number":176,"balance":18607,"firstname":"Kemp","lastname":"Walters","age":28,"gender":"F","address":"906 Howard Avenue","employer":"Eyewax","email":"kempwalters@eyewax.com","city":"Why","state":"KY","_is_real":true} +{"index":{"_id":"183"}} +{"account_number":183,"balance":14223,"firstname":"Hudson","lastname":"English","age":26,"gender":"F","address":"823 Herkimer Place","employer":"Xinware","email":"hudsonenglish@xinware.com","city":"Robbins","state":"ND","_is_real":true} +{"index":{"_id":"188"}} +{"account_number":188,"balance":41504,"firstname":"Tia","lastname":"Miranda","age":24,"gender":"F","address":"583 Ainslie Street","employer":"Jasper","email":"tiamiranda@jasper.com","city":"Summerset","state":"UT","_is_real":true} +{"index":{"_id":"190"}} +{"account_number":190,"balance":3150,"firstname":"Blake","lastname":"Davidson","age":30,"gender":"F","address":"636 Diamond Street","employer":"Quantasis","email":"blakedavidson@quantasis.com","city":"Crumpler","state":"KY","_is_real":true} +{"index":{"_id":"195"}} +{"account_number":195,"balance":5025,"firstname":"Kaye","lastname":"Gibson","age":31,"gender":"M","address":"955 Hopkins Street","employer":"Zork","email":"kayegibson@zork.com","city":"Ola","state":"WY","_is_real":true} +{"index":{"_id":"203"}} +{"account_number":203,"balance":21890,"firstname":"Eve","lastname":"Wyatt","age":33,"gender":"M","address":"435 Furman Street","employer":"Assitia","email":"evewyatt@assitia.com","city":"Jamestown","state":"MN","_is_real":true} +{"index":{"_id":"208"}} +{"account_number":208,"balance":40760,"firstname":"Garcia","lastname":"Hess","age":26,"gender":"F","address":"810 Nostrand Avenue","employer":"Quiltigen","email":"garciahess@quiltigen.com","city":"Brooktrails","state":"GA","_is_real":true} +{"index":{"_id":"210"}} +{"account_number":210,"balance":33946,"firstname":"Cherry","lastname":"Carey","age":24,"gender":"M","address":"539 Tiffany Place","employer":"Martgo","email":"cherrycarey@martgo.com","city":"Fairacres","state":"AK","_is_real":true} +{"index":{"_id":"215"}} +{"account_number":215,"balance":37427,"firstname":"Copeland","lastname":"Solomon","age":20,"gender":"M","address":"741 McDonald Avenue","employer":"Recognia","email":"copelandsolomon@recognia.com","city":"Edmund","state":"ME","_is_real":true} +{"index":{"_id":"222"}} +{"account_number":222,"balance":14764,"firstname":"Rachelle","lastname":"Rice","age":36,"gender":"M","address":"333 Narrows Avenue","employer":"Enaut","email":"rachellerice@enaut.com","city":"Wright","state":"AZ","_is_real":true} +{"index":{"_id":"227"}} +{"account_number":227,"balance":19780,"firstname":"Coleman","lastname":"Berg","age":22,"gender":"M","address":"776 Little Street","employer":"Exoteric","email":"colemanberg@exoteric.com","city":"Eagleville","state":"WV","_is_real":true} +{"index":{"_id":"234"}} +{"account_number":234,"balance":44207,"firstname":"Betty","lastname":"Hall","age":37,"gender":"F","address":"709 Garfield Place","employer":"Miraclis","email":"bettyhall@miraclis.com","city":"Bendon","state":"NY","_is_real":true} +{"index":{"_id":"239"}} +{"account_number":239,"balance":25719,"firstname":"Chang","lastname":"Boyer","age":36,"gender":"M","address":"895 Brigham Street","employer":"Qaboos","email":"changboyer@qaboos.com","city":"Belgreen","state":"NH","_is_real":true} +{"index":{"_id":"241"}} +{"account_number":241,"balance":25379,"firstname":"Schroeder","lastname":"Harrington","age":26,"gender":"M","address":"610 Tapscott Avenue","employer":"Otherway","email":"schroederharrington@otherway.com","city":"Ebro","state":"TX","_is_real":true} +{"index":{"_id":"246"}} +{"account_number":246,"balance":28405,"firstname":"Katheryn","lastname":"Foster","age":21,"gender":"F","address":"259 Kane Street","employer":"Quantalia","email":"katherynfoster@quantalia.com","city":"Bath","state":"TX","_is_real":true} +{"index":{"_id":"253"}} +{"account_number":253,"balance":20240,"firstname":"Melissa","lastname":"Gould","age":31,"gender":"M","address":"440 Fuller Place","employer":"Buzzopia","email":"melissagould@buzzopia.com","city":"Lumberton","state":"MD","_is_real":true} +{"index":{"_id":"258"}} +{"account_number":258,"balance":5712,"firstname":"Lindsey","lastname":"Hawkins","age":37,"gender":"M","address":"706 Frost Street","employer":"Enormo","email":"lindseyhawkins@enormo.com","city":"Gardners","state":"AK","_is_real":true} +{"index":{"_id":"260"}} +{"account_number":260,"balance":2726,"firstname":"Kari","lastname":"Skinner","age":30,"gender":"F","address":"735 Losee Terrace","employer":"Singavera","email":"kariskinner@singavera.com","city":"Rushford","state":"WV","_is_real":true} +{"index":{"_id":"265"}} +{"account_number":265,"balance":46910,"firstname":"Marion","lastname":"Schneider","age":26,"gender":"F","address":"574 Everett Avenue","employer":"Evidends","email":"marionschneider@evidends.com","city":"Maplewood","state":"WY","_is_real":true} +{"index":{"_id":"272"}} +{"account_number":272,"balance":19253,"firstname":"Lilly","lastname":"Morgan","age":25,"gender":"F","address":"689 Fleet Street","employer":"Biolive","email":"lillymorgan@biolive.com","city":"Sunbury","state":"OH","_is_real":true} +{"index":{"_id":"277"}} +{"account_number":277,"balance":29564,"firstname":"Romero","lastname":"Lott","age":31,"gender":"M","address":"456 Danforth Street","employer":"Plasto","email":"romerolott@plasto.com","city":"Vincent","state":"VT","_is_real":true} +{"index":{"_id":"284"}} +{"account_number":284,"balance":22806,"firstname":"Randolph","lastname":"Banks","age":29,"gender":"M","address":"875 Hamilton Avenue","employer":"Caxt","email":"randolphbanks@caxt.com","city":"Crawfordsville","state":"WA","_is_real":true} +{"index":{"_id":"289"}} +{"account_number":289,"balance":7798,"firstname":"Blair","lastname":"Church","age":29,"gender":"M","address":"370 Sutton Street","employer":"Cubix","email":"blairchurch@cubix.com","city":"Nile","state":"NH","_is_real":true} +{"index":{"_id":"291"}} +{"account_number":291,"balance":19955,"firstname":"Lynn","lastname":"Pollard","age":40,"gender":"F","address":"685 Pierrepont Street","employer":"Slambda","email":"lynnpollard@slambda.com","city":"Mappsville","state":"ID","_is_real":true} +{"index":{"_id":"296"}} +{"account_number":296,"balance":24606,"firstname":"Rosa","lastname":"Oliver","age":34,"gender":"M","address":"168 Woodbine Street","employer":"Idetica","email":"rosaoliver@idetica.com","city":"Robinson","state":"WY","_is_real":true} +{"index":{"_id":"304"}} +{"account_number":304,"balance":28647,"firstname":"Palmer","lastname":"Clark","age":35,"gender":"M","address":"866 Boulevard Court","employer":"Maximind","email":"palmerclark@maximind.com","city":"Avalon","state":"NH","_is_real":true} +{"index":{"_id":"309"}} +{"account_number":309,"balance":3830,"firstname":"Rosemarie","lastname":"Nieves","age":30,"gender":"M","address":"206 Alice Court","employer":"Zounds","email":"rosemarienieves@zounds.com","city":"Ferney","state":"AR","_is_real":true} +{"index":{"_id":"311"}} +{"account_number":311,"balance":13388,"firstname":"Vinson","lastname":"Ballard","age":23,"gender":"F","address":"960 Glendale Court","employer":"Gynk","email":"vinsonballard@gynk.com","city":"Fairforest","state":"WY","_is_real":true} +{"index":{"_id":"316"}} +{"account_number":316,"balance":8214,"firstname":"Anita","lastname":"Ewing","age":32,"gender":"M","address":"396 Lombardy Street","employer":"Panzent","email":"anitaewing@panzent.com","city":"Neahkahnie","state":"WY","_is_real":true} +{"index":{"_id":"323"}} +{"account_number":323,"balance":42230,"firstname":"Chelsea","lastname":"Gamble","age":34,"gender":"F","address":"356 Dare Court","employer":"Isosphere","email":"chelseagamble@isosphere.com","city":"Dundee","state":"MD","_is_real":true} +{"index":{"_id":"328"}} +{"account_number":328,"balance":12523,"firstname":"Good","lastname":"Campbell","age":27,"gender":"F","address":"438 Hicks Street","employer":"Gracker","email":"goodcampbell@gracker.com","city":"Marion","state":"CA","_is_real":true} +{"index":{"_id":"330"}} +{"account_number":330,"balance":41620,"firstname":"Yvette","lastname":"Browning","age":34,"gender":"F","address":"431 Beekman Place","employer":"Marketoid","email":"yvettebrowning@marketoid.com","city":"Talpa","state":"CO","_is_real":true} +{"index":{"_id":"335"}} +{"account_number":335,"balance":35433,"firstname":"Vera","lastname":"Hansen","age":24,"gender":"M","address":"252 Bushwick Avenue","employer":"Zanilla","email":"verahansen@zanilla.com","city":"Manila","state":"TN","_is_real":true} +{"index":{"_id":"342"}} +{"account_number":342,"balance":33670,"firstname":"Vivian","lastname":"Wells","age":36,"gender":"M","address":"570 Cobek Court","employer":"Nutralab","email":"vivianwells@nutralab.com","city":"Fontanelle","state":"OK","_is_real":true} +{"index":{"_id":"347"}} +{"account_number":347,"balance":36038,"firstname":"Gould","lastname":"Carson","age":24,"gender":"F","address":"784 Pulaski Street","employer":"Mobildata","email":"gouldcarson@mobildata.com","city":"Goochland","state":"MI","_is_real":true} +{"index":{"_id":"354"}} +{"account_number":354,"balance":21294,"firstname":"Kidd","lastname":"Mclean","age":22,"gender":"M","address":"691 Saratoga Avenue","employer":"Ronbert","email":"kiddmclean@ronbert.com","city":"Tioga","state":"ME","_is_real":true} +{"index":{"_id":"359"}} +{"account_number":359,"balance":29927,"firstname":"Vanessa","lastname":"Harvey","age":28,"gender":"F","address":"679 Rutledge Street","employer":"Zentime","email":"vanessaharvey@zentime.com","city":"Williston","state":"IL","_is_real":true} +{"index":{"_id":"361"}} +{"account_number":361,"balance":23659,"firstname":"Noreen","lastname":"Shelton","age":36,"gender":"M","address":"702 Tillary Street","employer":"Medmex","email":"noreenshelton@medmex.com","city":"Derwood","state":"NH","_is_real":true} +{"index":{"_id":"366"}} +{"account_number":366,"balance":42368,"firstname":"Lydia","lastname":"Cooke","age":31,"gender":"M","address":"470 Coleman Street","employer":"Comstar","email":"lydiacooke@comstar.com","city":"Datil","state":"TN","_is_real":true} +{"index":{"_id":"373"}} +{"account_number":373,"balance":9671,"firstname":"Simpson","lastname":"Carpenter","age":21,"gender":"M","address":"837 Horace Court","employer":"Snips","email":"simpsoncarpenter@snips.com","city":"Tolu","state":"MA","_is_real":true} +{"index":{"_id":"378"}} +{"account_number":378,"balance":27100,"firstname":"Watson","lastname":"Simpson","age":36,"gender":"F","address":"644 Thomas Street","employer":"Wrapture","email":"watsonsimpson@wrapture.com","city":"Keller","state":"TX","_is_real":true} +{"index":{"_id":"380"}} +{"account_number":380,"balance":35628,"firstname":"Fernandez","lastname":"Reid","age":33,"gender":"F","address":"154 Melba Court","employer":"Cosmosis","email":"fernandezreid@cosmosis.com","city":"Boyd","state":"NE","_is_real":true} +{"index":{"_id":"385"}} +{"account_number":385,"balance":11022,"firstname":"Rosalinda","lastname":"Valencia","age":22,"gender":"M","address":"933 Lloyd Street","employer":"Zoarere","email":"rosalindavalencia@zoarere.com","city":"Waverly","state":"GA","_is_real":true} +{"index":{"_id":"392"}} +{"account_number":392,"balance":31613,"firstname":"Dotson","lastname":"Dean","age":35,"gender":"M","address":"136 Ford Street","employer":"Petigems","email":"dotsondean@petigems.com","city":"Chical","state":"SD","_is_real":true} +{"index":{"_id":"397"}} +{"account_number":397,"balance":37418,"firstname":"Leonard","lastname":"Gray","age":36,"gender":"F","address":"840 Morgan Avenue","employer":"Recritube","email":"leonardgray@recritube.com","city":"Edenburg","state":"AL","_is_real":true} +{"index":{"_id":"400"}} +{"account_number":400,"balance":20685,"firstname":"Kane","lastname":"King","age":21,"gender":"F","address":"405 Cornelia Street","employer":"Tri@Tribalog","email":"kaneking@tri@tribalog.com","city":"Gulf","state":"VT","_is_real":true} +{"index":{"_id":"405"}} +{"account_number":405,"balance":5679,"firstname":"Strickland","lastname":"Fuller","age":26,"gender":"M","address":"990 Concord Street","employer":"Digique","email":"stricklandfuller@digique.com","city":"Southmont","state":"NV","_is_real":true} +{"index":{"_id":"412"}} +{"account_number":412,"balance":27436,"firstname":"Ilene","lastname":"Abbott","age":26,"gender":"M","address":"846 Vine Street","employer":"Typhonica","email":"ileneabbott@typhonica.com","city":"Cedarville","state":"VT","_is_real":true} +{"index":{"_id":"417"}} +{"account_number":417,"balance":1788,"firstname":"Wheeler","lastname":"Ayers","age":35,"gender":"F","address":"677 Hope Street","employer":"Fortean","email":"wheelerayers@fortean.com","city":"Ironton","state":"PA","_is_real":true} +{"index":{"_id":"424"}} +{"account_number":424,"balance":36818,"firstname":"Tracie","lastname":"Gregory","age":34,"gender":"M","address":"112 Hunterfly Place","employer":"Comstruct","email":"traciegregory@comstruct.com","city":"Onton","state":"TN","_is_real":true} +{"index":{"_id":"429"}} +{"account_number":429,"balance":46970,"firstname":"Cantu","lastname":"Lindsey","age":31,"gender":"M","address":"404 Willoughby Avenue","employer":"Inquala","email":"cantulindsey@inquala.com","city":"Cowiche","state":"IA","_is_real":true} +{"index":{"_id":"431"}} +{"account_number":431,"balance":13136,"firstname":"Laurie","lastname":"Shaw","age":26,"gender":"F","address":"263 Aviation Road","employer":"Zillanet","email":"laurieshaw@zillanet.com","city":"Harmon","state":"WV","_is_real":true} +{"index":{"_id":"436"}} +{"account_number":436,"balance":27585,"firstname":"Alexander","lastname":"Sargent","age":23,"gender":"M","address":"363 Albemarle Road","employer":"Fangold","email":"alexandersargent@fangold.com","city":"Calpine","state":"OR","_is_real":true} +{"index":{"_id":"443"}} +{"account_number":443,"balance":7588,"firstname":"Huff","lastname":"Thomas","age":23,"gender":"M","address":"538 Erskine Loop","employer":"Accufarm","email":"huffthomas@accufarm.com","city":"Corinne","state":"AL","_is_real":true} +{"index":{"_id":"448"}} +{"account_number":448,"balance":22776,"firstname":"Adriana","lastname":"Mcfadden","age":35,"gender":"F","address":"984 Woodside Avenue","employer":"Telequiet","email":"adrianamcfadden@telequiet.com","city":"Darrtown","state":"WI","_is_real":true} +{"index":{"_id":"450"}} +{"account_number":450,"balance":2643,"firstname":"Bradford","lastname":"Nielsen","age":25,"gender":"M","address":"487 Keen Court","employer":"Exovent","email":"bradfordnielsen@exovent.com","city":"Hamilton","state":"DE","_is_real":true} +{"index":{"_id":"455"}} +{"account_number":455,"balance":39556,"firstname":"Lynn","lastname":"Tran","age":36,"gender":"M","address":"741 Richmond Street","employer":"Optyk","email":"lynntran@optyk.com","city":"Clinton","state":"WV","_is_real":true} +{"index":{"_id":"462"}} +{"account_number":462,"balance":10871,"firstname":"Calderon","lastname":"Day","age":27,"gender":"M","address":"810 Milford Street","employer":"Cofine","email":"calderonday@cofine.com","city":"Kula","state":"OK","_is_real":true} +{"index":{"_id":"467"}} +{"account_number":467,"balance":6312,"firstname":"Angelica","lastname":"May","age":32,"gender":"F","address":"384 Karweg Place","employer":"Keeg","email":"angelicamay@keeg.com","city":"Tetherow","state":"IA","_is_real":true} +{"index":{"_id":"474"}} +{"account_number":474,"balance":35896,"firstname":"Obrien","lastname":"Walton","age":40,"gender":"F","address":"192 Ide Court","employer":"Suremax","email":"obrienwalton@suremax.com","city":"Crucible","state":"UT","_is_real":true} +{"index":{"_id":"479"}} +{"account_number":479,"balance":31865,"firstname":"Cameron","lastname":"Ross","age":40,"gender":"M","address":"904 Bouck Court","employer":"Telpod","email":"cameronross@telpod.com","city":"Nord","state":"MO","_is_real":true} +{"index":{"_id":"481"}} +{"account_number":481,"balance":20024,"firstname":"Lina","lastname":"Stanley","age":33,"gender":"M","address":"361 Hanover Place","employer":"Strozen","email":"linastanley@strozen.com","city":"Wyoming","state":"NC","_is_real":true} +{"index":{"_id":"486"}} +{"account_number":486,"balance":35902,"firstname":"Dixie","lastname":"Fuentes","age":22,"gender":"F","address":"991 Applegate Court","employer":"Portico","email":"dixiefuentes@portico.com","city":"Salix","state":"VA","_is_real":true} +{"index":{"_id":"493"}} +{"account_number":493,"balance":5871,"firstname":"Campbell","lastname":"Best","age":24,"gender":"M","address":"297 Friel Place","employer":"Fanfare","email":"campbellbest@fanfare.com","city":"Kidder","state":"GA","_is_real":true} +{"index":{"_id":"498"}} +{"account_number":498,"balance":10516,"firstname":"Stella","lastname":"Hinton","age":39,"gender":"F","address":"649 Columbia Place","employer":"Flyboyz","email":"stellahinton@flyboyz.com","city":"Crenshaw","state":"SC","_is_real":true} +{"index":{"_id":"501"}} +{"account_number":501,"balance":16572,"firstname":"Kelley","lastname":"Ochoa","age":36,"gender":"M","address":"451 Clifton Place","employer":"Bluplanet","email":"kelleyochoa@bluplanet.com","city":"Gouglersville","state":"CT","_is_real":true} +{"index":{"_id":"506"}} +{"account_number":506,"balance":43440,"firstname":"Davidson","lastname":"Salas","age":28,"gender":"M","address":"731 Cleveland Street","employer":"Sequitur","email":"davidsonsalas@sequitur.com","city":"Lloyd","state":"ME","_is_real":true} +{"index":{"_id":"513"}} +{"account_number":513,"balance":30040,"firstname":"Maryellen","lastname":"Rose","age":37,"gender":"F","address":"428 Durland Place","employer":"Waterbaby","email":"maryellenrose@waterbaby.com","city":"Kiskimere","state":"RI","_is_real":true} +{"index":{"_id":"518"}} +{"account_number":518,"balance":48954,"firstname":"Finch","lastname":"Curtis","age":29,"gender":"F","address":"137 Ryder Street","employer":"Viagrand","email":"finchcurtis@viagrand.com","city":"Riverton","state":"MO","_is_real":true} +{"index":{"_id":"520"}} +{"account_number":520,"balance":27987,"firstname":"Brandy","lastname":"Calhoun","age":32,"gender":"M","address":"818 Harden Street","employer":"Maxemia","email":"brandycalhoun@maxemia.com","city":"Sidman","state":"OR","_is_real":true} +{"index":{"_id":"525"}} +{"account_number":525,"balance":23545,"firstname":"Holly","lastname":"Miles","age":25,"gender":"M","address":"746 Ludlam Place","employer":"Xurban","email":"hollymiles@xurban.com","city":"Harold","state":"AR","_is_real":true} +{"index":{"_id":"532"}} +{"account_number":532,"balance":17207,"firstname":"Hardin","lastname":"Kirk","age":26,"gender":"M","address":"268 Canarsie Road","employer":"Exposa","email":"hardinkirk@exposa.com","city":"Stouchsburg","state":"IL","_is_real":true} +{"index":{"_id":"537"}} +{"account_number":537,"balance":31069,"firstname":"Morin","lastname":"Frost","age":29,"gender":"M","address":"910 Lake Street","employer":"Primordia","email":"morinfrost@primordia.com","city":"Rivera","state":"DE","_is_real":true} +{"index":{"_id":"544"}} +{"account_number":544,"balance":41735,"firstname":"Short","lastname":"Dennis","age":21,"gender":"F","address":"908 Glen Street","employer":"Minga","email":"shortdennis@minga.com","city":"Dale","state":"KY","_is_real":true} +{"index":{"_id":"549"}} +{"account_number":549,"balance":1932,"firstname":"Jacqueline","lastname":"Maxwell","age":40,"gender":"M","address":"444 Schenck Place","employer":"Fuelworks","email":"jacquelinemaxwell@fuelworks.com","city":"Oretta","state":"OR","_is_real":true} +{"index":{"_id":"551"}} +{"account_number":551,"balance":21732,"firstname":"Milagros","lastname":"Travis","age":27,"gender":"F","address":"380 Murdock Court","employer":"Sloganaut","email":"milagrostravis@sloganaut.com","city":"Homeland","state":"AR","_is_real":true} +{"index":{"_id":"556"}} +{"account_number":556,"balance":36420,"firstname":"Collier","lastname":"Odonnell","age":35,"gender":"M","address":"591 Nolans Lane","employer":"Sultraxin","email":"collierodonnell@sultraxin.com","city":"Fulford","state":"MD","_is_real":true} +{"index":{"_id":"563"}} +{"account_number":563,"balance":43403,"firstname":"Morgan","lastname":"Torres","age":30,"gender":"F","address":"672 Belvidere Street","employer":"Quonata","email":"morgantorres@quonata.com","city":"Hollymead","state":"KY","_is_real":true} +{"index":{"_id":"568"}} +{"account_number":568,"balance":36628,"firstname":"Lesa","lastname":"Maynard","age":29,"gender":"F","address":"295 Whitty Lane","employer":"Coash","email":"lesamaynard@coash.com","city":"Broadlands","state":"VT","_is_real":true} +{"index":{"_id":"570"}} +{"account_number":570,"balance":26751,"firstname":"Church","lastname":"Mercado","age":24,"gender":"F","address":"892 Wyckoff Street","employer":"Xymonk","email":"churchmercado@xymonk.com","city":"Gloucester","state":"KY","_is_real":true} +{"index":{"_id":"575"}} +{"account_number":575,"balance":12588,"firstname":"Buchanan","lastname":"Pope","age":39,"gender":"M","address":"581 Sumner Place","employer":"Stucco","email":"buchananpope@stucco.com","city":"Ellerslie","state":"MD","_is_real":true} +{"index":{"_id":"582"}} +{"account_number":582,"balance":33371,"firstname":"Manning","lastname":"Guthrie","age":24,"gender":"F","address":"271 Jodie Court","employer":"Xerex","email":"manningguthrie@xerex.com","city":"Breinigsville","state":"NM","_is_real":true} +{"index":{"_id":"587"}} +{"account_number":587,"balance":3468,"firstname":"Carly","lastname":"Johns","age":33,"gender":"M","address":"390 Noll Street","employer":"Gallaxia","email":"carlyjohns@gallaxia.com","city":"Emison","state":"DC","_is_real":true} +{"index":{"_id":"594"}} +{"account_number":594,"balance":28194,"firstname":"Golden","lastname":"Donovan","age":26,"gender":"M","address":"199 Jewel Street","employer":"Organica","email":"goldendonovan@organica.com","city":"Macdona","state":"RI","_is_real":true} +{"index":{"_id":"599"}} +{"account_number":599,"balance":11944,"firstname":"Joanna","lastname":"Jennings","age":36,"gender":"F","address":"318 Irving Street","employer":"Extremo","email":"joannajennings@extremo.com","city":"Bartley","state":"MI","_is_real":true} +{"index":{"_id":"602"}} +{"account_number":602,"balance":38699,"firstname":"Mcgowan","lastname":"Mcclain","age":33,"gender":"M","address":"361 Stoddard Place","employer":"Oatfarm","email":"mcgowanmcclain@oatfarm.com","city":"Kapowsin","state":"MI","_is_real":true} +{"index":{"_id":"607"}} +{"account_number":607,"balance":38350,"firstname":"White","lastname":"Small","age":38,"gender":"F","address":"736 Judge Street","employer":"Immunics","email":"whitesmall@immunics.com","city":"Fairfield","state":"HI","_is_real":true} +{"index":{"_id":"614"}} +{"account_number":614,"balance":13157,"firstname":"Salazar","lastname":"Howard","age":35,"gender":"F","address":"847 Imlay Street","employer":"Retrack","email":"salazarhoward@retrack.com","city":"Grill","state":"FL","_is_real":true} +{"index":{"_id":"619"}} +{"account_number":619,"balance":48755,"firstname":"Grimes","lastname":"Reynolds","age":36,"gender":"M","address":"378 Denton Place","employer":"Frenex","email":"grimesreynolds@frenex.com","city":"Murillo","state":"LA","_is_real":true} +{"index":{"_id":"621"}} +{"account_number":621,"balance":35480,"firstname":"Leslie","lastname":"Sloan","age":26,"gender":"F","address":"336 Kansas Place","employer":"Dancity","email":"lesliesloan@dancity.com","city":"Corriganville","state":"AR","_is_real":true} +{"index":{"_id":"626"}} +{"account_number":626,"balance":19498,"firstname":"Ava","lastname":"Richardson","age":31,"gender":"F","address":"666 Nautilus Avenue","employer":"Cinaster","email":"avarichardson@cinaster.com","city":"Sutton","state":"AL","_is_real":true} +{"index":{"_id":"633"}} +{"account_number":633,"balance":35874,"firstname":"Conner","lastname":"Ramos","age":34,"gender":"M","address":"575 Agate Court","employer":"Insource","email":"connerramos@insource.com","city":"Madaket","state":"OK","_is_real":true} +{"index":{"_id":"638"}} +{"account_number":638,"balance":2658,"firstname":"Bridget","lastname":"Gallegos","age":31,"gender":"M","address":"383 Wogan Terrace","employer":"Songlines","email":"bridgetgallegos@songlines.com","city":"Linganore","state":"WA","_is_real":true} +{"index":{"_id":"640"}} +{"account_number":640,"balance":35596,"firstname":"Candace","lastname":"Hancock","age":25,"gender":"M","address":"574 Riverdale Avenue","employer":"Animalia","email":"candacehancock@animalia.com","city":"Blandburg","state":"KY","_is_real":true} +{"index":{"_id":"645"}} +{"account_number":645,"balance":29362,"firstname":"Edwina","lastname":"Hutchinson","age":26,"gender":"F","address":"892 Pacific Street","employer":"Essensia","email":"edwinahutchinson@essensia.com","city":"Dowling","state":"NE","_is_real":true} +{"index":{"_id":"652"}} +{"account_number":652,"balance":17363,"firstname":"Bonner","lastname":"Garner","age":26,"gender":"M","address":"219 Grafton Street","employer":"Utarian","email":"bonnergarner@utarian.com","city":"Vandiver","state":"PA","_is_real":true} +{"index":{"_id":"657"}} +{"account_number":657,"balance":40475,"firstname":"Kathleen","lastname":"Wilder","age":34,"gender":"F","address":"286 Sutter Avenue","employer":"Solgan","email":"kathleenwilder@solgan.com","city":"Graniteville","state":"MI","_is_real":true} +{"index":{"_id":"664"}} +{"account_number":664,"balance":16163,"firstname":"Hart","lastname":"Mccormick","age":40,"gender":"M","address":"144 Guider Avenue","employer":"Dyno","email":"hartmccormick@dyno.com","city":"Carbonville","state":"ID","_is_real":true} +{"index":{"_id":"669"}} +{"account_number":669,"balance":16934,"firstname":"Jewel","lastname":"Estrada","age":28,"gender":"M","address":"896 Meeker Avenue","employer":"Zilla","email":"jewelestrada@zilla.com","city":"Goodville","state":"PA","_is_real":true} +{"index":{"_id":"671"}} +{"account_number":671,"balance":29029,"firstname":"Antoinette","lastname":"Cook","age":34,"gender":"M","address":"375 Cumberland Street","employer":"Harmoney","email":"antoinettecook@harmoney.com","city":"Bergoo","state":"VT","_is_real":true} +{"index":{"_id":"676"}} +{"account_number":676,"balance":23842,"firstname":"Lisa","lastname":"Dudley","age":34,"gender":"M","address":"506 Vanderveer Street","employer":"Tropoli","email":"lisadudley@tropoli.com","city":"Konterra","state":"NY","_is_real":true} +{"index":{"_id":"683"}} +{"account_number":683,"balance":4381,"firstname":"Matilda","lastname":"Berger","age":39,"gender":"M","address":"884 Noble Street","employer":"Fibrodyne","email":"matildaberger@fibrodyne.com","city":"Shepardsville","state":"TN","_is_real":true} +{"index":{"_id":"688"}} +{"account_number":688,"balance":17931,"firstname":"Freeman","lastname":"Zamora","age":22,"gender":"F","address":"114 Herzl Street","employer":"Elemantra","email":"freemanzamora@elemantra.com","city":"Libertytown","state":"NM","_is_real":true} +{"index":{"_id":"690"}} +{"account_number":690,"balance":18127,"firstname":"Russo","lastname":"Swanson","age":35,"gender":"F","address":"256 Roebling Street","employer":"Zaj","email":"russoswanson@zaj.com","city":"Hoagland","state":"MI","_is_real":true} +{"index":{"_id":"695"}} +{"account_number":695,"balance":36800,"firstname":"Gonzales","lastname":"Mcfarland","age":26,"gender":"F","address":"647 Louisa Street","employer":"Songbird","email":"gonzalesmcfarland@songbird.com","city":"Crisman","state":"ID","_is_real":true} +{"index":{"_id":"703"}} +{"account_number":703,"balance":27443,"firstname":"Dona","lastname":"Burton","age":29,"gender":"M","address":"489 Flatlands Avenue","employer":"Cytrex","email":"donaburton@cytrex.com","city":"Reno","state":"VA","_is_real":true} +{"index":{"_id":"708"}} +{"account_number":708,"balance":34002,"firstname":"May","lastname":"Ortiz","age":28,"gender":"F","address":"244 Chauncey Street","employer":"Syntac","email":"mayortiz@syntac.com","city":"Munjor","state":"ID","_is_real":true} +{"index":{"_id":"710"}} +{"account_number":710,"balance":33650,"firstname":"Shelton","lastname":"Stark","age":37,"gender":"M","address":"404 Ovington Avenue","employer":"Kraggle","email":"sheltonstark@kraggle.com","city":"Ogema","state":"TN","_is_real":true} +{"index":{"_id":"715"}} +{"account_number":715,"balance":23734,"firstname":"Tammi","lastname":"Hodge","age":24,"gender":"M","address":"865 Church Lane","employer":"Netur","email":"tammihodge@netur.com","city":"Lacomb","state":"KS","_is_real":true} +{"index":{"_id":"722"}} +{"account_number":722,"balance":27256,"firstname":"Roberts","lastname":"Beasley","age":34,"gender":"F","address":"305 Kings Hwy","employer":"Quintity","email":"robertsbeasley@quintity.com","city":"Hayden","state":"PA","_is_real":true} +{"index":{"_id":"727"}} +{"account_number":727,"balance":27263,"firstname":"Natasha","lastname":"Knapp","age":36,"gender":"M","address":"723 Hubbard Street","employer":"Exostream","email":"natashaknapp@exostream.com","city":"Trexlertown","state":"LA","_is_real":true} +{"index":{"_id":"734"}} +{"account_number":734,"balance":20325,"firstname":"Keri","lastname":"Kinney","age":23,"gender":"M","address":"490 Balfour Place","employer":"Retrotex","email":"kerikinney@retrotex.com","city":"Salunga","state":"PA","_is_real":true} +{"index":{"_id":"739"}} +{"account_number":739,"balance":39063,"firstname":"Gwen","lastname":"Hardy","age":33,"gender":"F","address":"733 Stuart Street","employer":"Exozent","email":"gwenhardy@exozent.com","city":"Drytown","state":"NY","_is_real":true} +{"index":{"_id":"741"}} +{"account_number":741,"balance":33074,"firstname":"Nielsen","lastname":"Good","age":22,"gender":"M","address":"404 Norfolk Street","employer":"Kiggle","email":"nielsengood@kiggle.com","city":"Cumberland","state":"WA","_is_real":true} +{"index":{"_id":"746"}} +{"account_number":746,"balance":15970,"firstname":"Marguerite","lastname":"Wall","age":28,"gender":"F","address":"364 Crosby Avenue","employer":"Aquoavo","email":"margueritewall@aquoavo.com","city":"Jeff","state":"MI","_is_real":true} +{"index":{"_id":"753"}} +{"account_number":753,"balance":33340,"firstname":"Katina","lastname":"Alford","age":21,"gender":"F","address":"690 Ross Street","employer":"Intrawear","email":"katinaalford@intrawear.com","city":"Grimsley","state":"OK","_is_real":true} +{"index":{"_id":"758"}} +{"account_number":758,"balance":15739,"firstname":"Berta","lastname":"Short","age":28,"gender":"M","address":"149 Surf Avenue","employer":"Ozean","email":"bertashort@ozean.com","city":"Odessa","state":"UT","_is_real":true} +{"index":{"_id":"760"}} +{"account_number":760,"balance":40996,"firstname":"Rhea","lastname":"Blair","age":37,"gender":"F","address":"440 Hubbard Place","employer":"Bicol","email":"rheablair@bicol.com","city":"Stockwell","state":"LA","_is_real":true} +{"index":{"_id":"765"}} +{"account_number":765,"balance":31278,"firstname":"Knowles","lastname":"Cunningham","age":23,"gender":"M","address":"753 Macdougal Street","employer":"Thredz","email":"knowlescunningham@thredz.com","city":"Thomasville","state":"WA","_is_real":true} +{"index":{"_id":"772"}} +{"account_number":772,"balance":37849,"firstname":"Eloise","lastname":"Sparks","age":21,"gender":"M","address":"608 Willow Street","employer":"Satiance","email":"eloisesparks@satiance.com","city":"Richford","state":"NY","_is_real":true} +{"index":{"_id":"777"}} +{"account_number":777,"balance":48294,"firstname":"Adkins","lastname":"Mejia","age":32,"gender":"M","address":"186 Oxford Walk","employer":"Datagen","email":"adkinsmejia@datagen.com","city":"Faywood","state":"OK","_is_real":true} +{"index":{"_id":"784"}} +{"account_number":784,"balance":25291,"firstname":"Mabel","lastname":"Thornton","age":21,"gender":"M","address":"124 Louisiana Avenue","employer":"Zolavo","email":"mabelthornton@zolavo.com","city":"Lynn","state":"AL","_is_real":true} +{"index":{"_id":"789"}} +{"account_number":789,"balance":8760,"firstname":"Cunningham","lastname":"Kerr","age":27,"gender":"F","address":"154 Sharon Street","employer":"Polarium","email":"cunninghamkerr@polarium.com","city":"Tuskahoma","state":"MS","_is_real":true} +{"index":{"_id":"791"}} +{"account_number":791,"balance":48249,"firstname":"Janine","lastname":"Huber","age":38,"gender":"F","address":"348 Porter Avenue","employer":"Viocular","email":"janinehuber@viocular.com","city":"Fivepointville","state":"MA","_is_real":true} +{"index":{"_id":"796"}} +{"account_number":796,"balance":23503,"firstname":"Mona","lastname":"Craft","age":35,"gender":"F","address":"511 Henry Street","employer":"Opticom","email":"monacraft@opticom.com","city":"Websterville","state":"IN","_is_real":true} +{"index":{"_id":"804"}} +{"account_number":804,"balance":23610,"firstname":"Rojas","lastname":"Oneal","age":27,"gender":"M","address":"669 Sandford Street","employer":"Glukgluk","email":"rojasoneal@glukgluk.com","city":"Wheaton","state":"ME","_is_real":true} +{"index":{"_id":"809"}} +{"account_number":809,"balance":47812,"firstname":"Christie","lastname":"Strickland","age":30,"gender":"M","address":"346 Bancroft Place","employer":"Anarco","email":"christiestrickland@anarco.com","city":"Baden","state":"NV","_is_real":true} +{"index":{"_id":"811"}} +{"account_number":811,"balance":26007,"firstname":"Walls","lastname":"Rogers","age":28,"gender":"F","address":"352 Freeman Street","employer":"Geekmosis","email":"wallsrogers@geekmosis.com","city":"Caroleen","state":"NV","_is_real":true} +{"index":{"_id":"816"}} +{"account_number":816,"balance":9567,"firstname":"Cornelia","lastname":"Lane","age":20,"gender":"F","address":"384 Bainbridge Street","employer":"Sulfax","email":"cornelialane@sulfax.com","city":"Elizaville","state":"MS","_is_real":true} +{"index":{"_id":"823"}} +{"account_number":823,"balance":48726,"firstname":"Celia","lastname":"Bernard","age":33,"gender":"F","address":"466 Amboy Street","employer":"Mitroc","email":"celiabernard@mitroc.com","city":"Skyland","state":"GA","_is_real":true} +{"index":{"_id":"828"}} +{"account_number":828,"balance":44890,"firstname":"Blanche","lastname":"Holmes","age":33,"gender":"F","address":"605 Stryker Court","employer":"Motovate","email":"blancheholmes@motovate.com","city":"Loomis","state":"KS","_is_real":true} +{"index":{"_id":"830"}} +{"account_number":830,"balance":45210,"firstname":"Louella","lastname":"Chan","age":23,"gender":"M","address":"511 Heath Place","employer":"Conferia","email":"louellachan@conferia.com","city":"Brookfield","state":"OK","_is_real":true} +{"index":{"_id":"835"}} +{"account_number":835,"balance":46558,"firstname":"Glover","lastname":"Rutledge","age":25,"gender":"F","address":"641 Royce Street","employer":"Ginkogene","email":"gloverrutledge@ginkogene.com","city":"Dixonville","state":"VA","_is_real":true} +{"index":{"_id":"842"}} +{"account_number":842,"balance":49587,"firstname":"Meagan","lastname":"Buckner","age":23,"gender":"F","address":"833 Bushwick Court","employer":"Biospan","email":"meaganbuckner@biospan.com","city":"Craig","state":"TX","_is_real":true} +{"index":{"_id":"847"}} +{"account_number":847,"balance":8652,"firstname":"Antonia","lastname":"Duncan","age":23,"gender":"M","address":"644 Stryker Street","employer":"Talae","email":"antoniaduncan@talae.com","city":"Dawn","state":"MO","_is_real":true} +{"index":{"_id":"854"}} +{"account_number":854,"balance":49795,"firstname":"Jimenez","lastname":"Barry","age":25,"gender":"F","address":"603 Cooper Street","employer":"Verton","email":"jimenezbarry@verton.com","city":"Moscow","state":"AL","_is_real":true} +{"index":{"_id":"859"}} +{"account_number":859,"balance":20734,"firstname":"Beulah","lastname":"Stuart","age":24,"gender":"F","address":"651 Albemarle Terrace","employer":"Hatology","email":"beulahstuart@hatology.com","city":"Waiohinu","state":"RI","_is_real":true} +{"index":{"_id":"861"}} +{"account_number":861,"balance":44173,"firstname":"Jaime","lastname":"Wilson","age":35,"gender":"M","address":"680 Richardson Street","employer":"Temorak","email":"jaimewilson@temorak.com","city":"Fidelis","state":"FL","_is_real":true} +{"index":{"_id":"866"}} +{"account_number":866,"balance":45565,"firstname":"Araceli","lastname":"Woodward","age":28,"gender":"M","address":"326 Meadow Street","employer":"Olympix","email":"araceliwoodward@olympix.com","city":"Dana","state":"KS","_is_real":true} +{"index":{"_id":"873"}} +{"account_number":873,"balance":43931,"firstname":"Tisha","lastname":"Cotton","age":39,"gender":"F","address":"432 Lincoln Road","employer":"Buzzmaker","email":"tishacotton@buzzmaker.com","city":"Bluetown","state":"GA","_is_real":true} +{"index":{"_id":"878"}} +{"account_number":878,"balance":49159,"firstname":"Battle","lastname":"Blackburn","age":40,"gender":"F","address":"234 Hendrix Street","employer":"Zilphur","email":"battleblackburn@zilphur.com","city":"Wanamie","state":"PA","_is_real":true} +{"index":{"_id":"880"}} +{"account_number":880,"balance":22575,"firstname":"Christian","lastname":"Myers","age":35,"gender":"M","address":"737 Crown Street","employer":"Combogen","email":"christianmyers@combogen.com","city":"Abrams","state":"OK","_is_real":true} +{"index":{"_id":"885"}} +{"account_number":885,"balance":31661,"firstname":"Valdez","lastname":"Roberson","age":40,"gender":"F","address":"227 Scholes Street","employer":"Delphide","email":"valdezroberson@delphide.com","city":"Chilton","state":"MT","_is_real":true} +{"index":{"_id":"892"}} +{"account_number":892,"balance":44974,"firstname":"Hill","lastname":"Hayes","age":29,"gender":"M","address":"721 Dooley Street","employer":"Fuelton","email":"hillhayes@fuelton.com","city":"Orason","state":"MT","_is_real":true} +{"index":{"_id":"897"}} +{"account_number":897,"balance":45973,"firstname":"Alyson","lastname":"Irwin","age":25,"gender":"M","address":"731 Poplar Street","employer":"Quizka","email":"alysonirwin@quizka.com","city":"Singer","state":"VA","_is_real":true} +{"index":{"_id":"900"}} +{"account_number":900,"balance":6124,"firstname":"Gonzalez","lastname":"Watson","age":23,"gender":"M","address":"624 Sullivan Street","employer":"Marvane","email":"gonzalezwatson@marvane.com","city":"Wikieup","state":"IL","_is_real":true} +{"index":{"_id":"905"}} +{"account_number":905,"balance":29438,"firstname":"Schultz","lastname":"Moreno","age":20,"gender":"F","address":"761 Cedar Street","employer":"Paragonia","email":"schultzmoreno@paragonia.com","city":"Glenshaw","state":"SC","_is_real":true} +{"index":{"_id":"912"}} +{"account_number":912,"balance":13675,"firstname":"Flora","lastname":"Alvarado","age":26,"gender":"M","address":"771 Vandervoort Avenue","employer":"Boilicon","email":"floraalvarado@boilicon.com","city":"Vivian","state":"ID","_is_real":true} +{"index":{"_id":"917"}} +{"account_number":917,"balance":47782,"firstname":"Parks","lastname":"Hurst","age":24,"gender":"M","address":"933 Cozine Avenue","employer":"Pyramis","email":"parkshurst@pyramis.com","city":"Lindcove","state":"GA","_is_real":true} +{"index":{"_id":"924"}} +{"account_number":924,"balance":3811,"firstname":"Hilary","lastname":"Leonard","age":24,"gender":"M","address":"235 Hegeman Avenue","employer":"Metroz","email":"hilaryleonard@metroz.com","city":"Roosevelt","state":"ME","_is_real":true} +{"index":{"_id":"929"}} +{"account_number":929,"balance":34708,"firstname":"Willie","lastname":"Hickman","age":35,"gender":"M","address":"430 Devoe Street","employer":"Apextri","email":"williehickman@apextri.com","city":"Clay","state":"MS","_is_real":true} +{"index":{"_id":"931"}} +{"account_number":931,"balance":8244,"firstname":"Ingrid","lastname":"Garcia","age":23,"gender":"F","address":"674 Indiana Place","employer":"Balooba","email":"ingridgarcia@balooba.com","city":"Interlochen","state":"AZ","_is_real":true} +{"index":{"_id":"936"}} +{"account_number":936,"balance":22430,"firstname":"Beth","lastname":"Frye","age":36,"gender":"M","address":"462 Thatford Avenue","employer":"Puria","email":"bethfrye@puria.com","city":"Hiseville","state":"LA","_is_real":true} +{"index":{"_id":"943"}} +{"account_number":943,"balance":24187,"firstname":"Wagner","lastname":"Griffin","age":23,"gender":"M","address":"489 Ellery Street","employer":"Gazak","email":"wagnergriffin@gazak.com","city":"Lorraine","state":"HI","_is_real":true} +{"index":{"_id":"948"}} +{"account_number":948,"balance":37074,"firstname":"Sargent","lastname":"Powers","age":40,"gender":"M","address":"532 Fiske Place","employer":"Accuprint","email":"sargentpowers@accuprint.com","city":"Umapine","state":"AK","_is_real":true} +{"index":{"_id":"950"}} +{"account_number":950,"balance":30916,"firstname":"Sherrie","lastname":"Patel","age":32,"gender":"F","address":"658 Langham Street","employer":"Futurize","email":"sherriepatel@futurize.com","city":"Garfield","state":"OR","_is_real":true} +{"index":{"_id":"955"}} +{"account_number":955,"balance":41621,"firstname":"Klein","lastname":"Kemp","age":33,"gender":"M","address":"370 Vanderbilt Avenue","employer":"Synkgen","email":"kleinkemp@synkgen.com","city":"Bonanza","state":"FL","_is_real":true} +{"index":{"_id":"962"}} +{"account_number":962,"balance":32096,"firstname":"Trujillo","lastname":"Wilcox","age":21,"gender":"F","address":"914 Duffield Street","employer":"Extragene","email":"trujillowilcox@extragene.com","city":"Golconda","state":"MA","_is_real":true} +{"index":{"_id":"967"}} +{"account_number":967,"balance":19161,"firstname":"Carrie","lastname":"Huffman","age":36,"gender":"F","address":"240 Sands Street","employer":"Injoy","email":"carriehuffman@injoy.com","city":"Leroy","state":"CA","_is_real":true} +{"index":{"_id":"974"}} +{"account_number":974,"balance":38082,"firstname":"Deborah","lastname":"Yang","age":26,"gender":"F","address":"463 Goodwin Place","employer":"Entogrok","email":"deborahyang@entogrok.com","city":"Herald","state":"KY","_is_real":true} +{"index":{"_id":"979"}} +{"account_number":979,"balance":43130,"firstname":"Vaughn","lastname":"Pittman","age":29,"gender":"M","address":"446 Tompkins Place","employer":"Phormula","email":"vaughnpittman@phormula.com","city":"Fingerville","state":"WI","_is_real":true} +{"index":{"_id":"981"}} +{"account_number":981,"balance":20278,"firstname":"Nolan","lastname":"Warner","age":29,"gender":"F","address":"753 Channel Avenue","employer":"Interodeo","email":"nolanwarner@interodeo.com","city":"Layhill","state":"MT","_is_real":true} +{"index":{"_id":"986"}} +{"account_number":986,"balance":35086,"firstname":"Norris","lastname":"Hubbard","age":31,"gender":"M","address":"600 Celeste Court","employer":"Printspan","email":"norrishubbard@printspan.com","city":"Cassel","state":"MI","_is_real":true} +{"index":{"_id":"993"}} +{"account_number":993,"balance":26487,"firstname":"Campos","lastname":"Olsen","age":37,"gender":"M","address":"873 Covert Street","employer":"Isbol","email":"camposolsen@isbol.com","city":"Glendale","state":"AK","_is_real":true} +{"index":{"_id":"998"}} +{"account_number":998,"balance":16869,"firstname":"Letha","lastname":"Baker","age":40,"gender":"F","address":"206 Llama Court","employer":"Dognosis","email":"lethabaker@dognosis.com","city":"Dunlo","state":"WV","_is_real":true} +{"index":{"_id":"2"}} +{"account_number":2,"balance":28838,"firstname":"Roberta","lastname":"Bender","age":22,"gender":"F","address":"560 Kingsway Place","employer":"Chillium","email":"robertabender@chillium.com","city":"Bennett","state":"LA","_is_real":true} +{"index":{"_id":"7"}} +{"account_number":7,"balance":39121,"firstname":"Levy","lastname":"Richard","age":22,"gender":"M","address":"820 Logan Street","employer":"Teraprene","email":"levyrichard@teraprene.com","city":"Shrewsbury","state":"MO","_is_real":true} +{"index":{"_id":"14"}} +{"account_number":14,"balance":20480,"firstname":"Erma","lastname":"Kane","age":39,"gender":"F","address":"661 Vista Place","employer":"Stockpost","email":"ermakane@stockpost.com","city":"Chamizal","state":"NY","_is_real":true} +{"index":{"_id":"19"}} +{"account_number":19,"balance":27894,"firstname":"Schwartz","lastname":"Buchanan","age":28,"gender":"F","address":"449 Mersereau Court","employer":"Sybixtex","email":"schwartzbuchanan@sybixtex.com","city":"Greenwich","state":"KS","_is_real":true} +{"index":{"_id":"21"}} +{"account_number":21,"balance":7004,"firstname":"Estella","lastname":"Paul","age":38,"gender":"M","address":"859 Portal Street","employer":"Zillatide","email":"estellapaul@zillatide.com","city":"Churchill","state":"WV","_is_real":true} +{"index":{"_id":"26"}} +{"account_number":26,"balance":14127,"firstname":"Lorraine","lastname":"Mccullough","age":39,"gender":"F","address":"157 Dupont Street","employer":"Zosis","email":"lorrainemccullough@zosis.com","city":"Dennard","state":"NH","_is_real":true} +{"index":{"_id":"33"}} +{"account_number":33,"balance":35439,"firstname":"Savannah","lastname":"Kirby","age":30,"gender":"F","address":"372 Malta Street","employer":"Musanpoly","email":"savannahkirby@musanpoly.com","city":"Muse","state":"AK","_is_real":true} +{"index":{"_id":"38"}} +{"account_number":38,"balance":10511,"firstname":"Erna","lastname":"Fields","age":32,"gender":"M","address":"357 Maple Street","employer":"Eweville","email":"ernafields@eweville.com","city":"Twilight","state":"MS","_is_real":true} +{"index":{"_id":"40"}} +{"account_number":40,"balance":33882,"firstname":"Pace","lastname":"Molina","age":40,"gender":"M","address":"263 Ovington Court","employer":"Cytrak","email":"pacemolina@cytrak.com","city":"Silkworth","state":"OR","_is_real":true} +{"index":{"_id":"45"}} +{"account_number":45,"balance":44478,"firstname":"Geneva","lastname":"Morin","age":21,"gender":"F","address":"357 Herkimer Street","employer":"Ezent","email":"genevamorin@ezent.com","city":"Blanco","state":"AZ","_is_real":true} +{"index":{"_id":"52"}} +{"account_number":52,"balance":46425,"firstname":"Kayla","lastname":"Bradshaw","age":31,"gender":"M","address":"449 Barlow Drive","employer":"Magnemo","email":"kaylabradshaw@magnemo.com","city":"Wawona","state":"AZ","_is_real":true} +{"index":{"_id":"57"}} +{"account_number":57,"balance":8705,"firstname":"Powell","lastname":"Herring","age":21,"gender":"M","address":"263 Merit Court","employer":"Digiprint","email":"powellherring@digiprint.com","city":"Coral","state":"MT","_is_real":true} +{"index":{"_id":"64"}} +{"account_number":64,"balance":44036,"firstname":"Miles","lastname":"Battle","age":35,"gender":"F","address":"988 Homecrest Avenue","employer":"Koffee","email":"milesbattle@koffee.com","city":"Motley","state":"ID","_is_real":true} +{"index":{"_id":"69"}} +{"account_number":69,"balance":14253,"firstname":"Desiree","lastname":"Harrison","age":24,"gender":"M","address":"694 Garland Court","employer":"Barkarama","email":"desireeharrison@barkarama.com","city":"Hackneyville","state":"GA","_is_real":true} +{"index":{"_id":"71"}} +{"account_number":71,"balance":38201,"firstname":"Sharpe","lastname":"Hoffman","age":39,"gender":"F","address":"450 Conklin Avenue","employer":"Centree","email":"sharpehoffman@centree.com","city":"Urbana","state":"WY","_is_real":true} +{"index":{"_id":"76"}} +{"account_number":76,"balance":38345,"firstname":"Claudette","lastname":"Beard","age":24,"gender":"F","address":"748 Dorset Street","employer":"Repetwire","email":"claudettebeard@repetwire.com","city":"Caln","state":"TX","_is_real":true} +{"index":{"_id":"83"}} +{"account_number":83,"balance":35928,"firstname":"Mayo","lastname":"Cleveland","age":28,"gender":"M","address":"720 Brooklyn Road","employer":"Indexia","email":"mayocleveland@indexia.com","city":"Roberts","state":"ND","_is_real":true} +{"index":{"_id":"88"}} +{"account_number":88,"balance":26418,"firstname":"Adela","lastname":"Tyler","age":21,"gender":"F","address":"737 Clove Road","employer":"Surelogic","email":"adelatyler@surelogic.com","city":"Boling","state":"SD","_is_real":true} +{"index":{"_id":"90"}} +{"account_number":90,"balance":25332,"firstname":"Herman","lastname":"Snyder","age":22,"gender":"F","address":"737 College Place","employer":"Lunchpod","email":"hermansnyder@lunchpod.com","city":"Flintville","state":"IA","_is_real":true} +{"index":{"_id":"95"}} +{"account_number":95,"balance":1650,"firstname":"Dominguez","lastname":"Le","age":20,"gender":"M","address":"539 Grace Court","employer":"Portica","email":"dominguezle@portica.com","city":"Wollochet","state":"KS","_is_real":true} +{"index":{"_id":"103"}} +{"account_number":103,"balance":11253,"firstname":"Calhoun","lastname":"Bruce","age":33,"gender":"F","address":"731 Clarkson Avenue","employer":"Automon","email":"calhounbruce@automon.com","city":"Marienthal","state":"IL","_is_real":true} +{"index":{"_id":"108"}} +{"account_number":108,"balance":19015,"firstname":"Christensen","lastname":"Weaver","age":21,"gender":"M","address":"398 Dearborn Court","employer":"Quilk","email":"christensenweaver@quilk.com","city":"Belvoir","state":"TX","_is_real":true} +{"index":{"_id":"110"}} +{"account_number":110,"balance":4850,"firstname":"Daphne","lastname":"Byrd","age":23,"gender":"F","address":"239 Conover Street","employer":"Freakin","email":"daphnebyrd@freakin.com","city":"Taft","state":"MN","_is_real":true} +{"index":{"_id":"115"}} +{"account_number":115,"balance":18750,"firstname":"Nikki","lastname":"Doyle","age":31,"gender":"F","address":"537 Clara Street","employer":"Fossiel","email":"nikkidoyle@fossiel.com","city":"Caron","state":"MS","_is_real":true} +{"index":{"_id":"122"}} +{"account_number":122,"balance":17128,"firstname":"Aurora","lastname":"Fry","age":31,"gender":"F","address":"227 Knapp Street","employer":"Makingway","email":"aurorafry@makingway.com","city":"Maybell","state":"NE","_is_real":true} +{"index":{"_id":"127"}} +{"account_number":127,"balance":48734,"firstname":"Diann","lastname":"Mclaughlin","age":33,"gender":"F","address":"340 Clermont Avenue","employer":"Enomen","email":"diannmclaughlin@enomen.com","city":"Rutherford","state":"ND","_is_real":true} +{"index":{"_id":"134"}} +{"account_number":134,"balance":33829,"firstname":"Madelyn","lastname":"Norris","age":30,"gender":"F","address":"176 Noel Avenue","employer":"Endicil","email":"madelynnorris@endicil.com","city":"Walker","state":"NE","_is_real":true} +{"index":{"_id":"139"}} +{"account_number":139,"balance":18444,"firstname":"Rios","lastname":"Todd","age":35,"gender":"F","address":"281 Georgia Avenue","employer":"Uberlux","email":"riostodd@uberlux.com","city":"Hannasville","state":"PA","_is_real":true} +{"index":{"_id":"141"}} +{"account_number":141,"balance":20790,"firstname":"Liliana","lastname":"Caldwell","age":29,"gender":"M","address":"414 Huron Street","employer":"Rubadub","email":"lilianacaldwell@rubadub.com","city":"Hiwasse","state":"OK","_is_real":true} +{"index":{"_id":"146"}} +{"account_number":146,"balance":39078,"firstname":"Lang","lastname":"Kaufman","age":32,"gender":"F","address":"626 Beverley Road","employer":"Rodeomad","email":"langkaufman@rodeomad.com","city":"Mahtowa","state":"RI","_is_real":true} +{"index":{"_id":"153"}} +{"account_number":153,"balance":32074,"firstname":"Bird","lastname":"Cochran","age":31,"gender":"F","address":"691 Bokee Court","employer":"Supremia","email":"birdcochran@supremia.com","city":"Barrelville","state":"NE","_is_real":true} +{"index":{"_id":"158"}} +{"account_number":158,"balance":9380,"firstname":"Natalie","lastname":"Mcdowell","age":27,"gender":"M","address":"953 Roder Avenue","employer":"Myopium","email":"nataliemcdowell@myopium.com","city":"Savage","state":"ND","_is_real":true} +{"index":{"_id":"160"}} +{"account_number":160,"balance":48974,"firstname":"Hull","lastname":"Cherry","age":23,"gender":"F","address":"275 Beaumont Street","employer":"Noralex","email":"hullcherry@noralex.com","city":"Whipholt","state":"WA","_is_real":true} +{"index":{"_id":"165"}} +{"account_number":165,"balance":18956,"firstname":"Sims","lastname":"Mckay","age":40,"gender":"F","address":"205 Jackson Street","employer":"Comtour","email":"simsmckay@comtour.com","city":"Tilden","state":"DC","_is_real":true} +{"index":{"_id":"172"}} +{"account_number":172,"balance":18356,"firstname":"Marie","lastname":"Whitehead","age":20,"gender":"M","address":"704 Monaco Place","employer":"Sultrax","email":"mariewhitehead@sultrax.com","city":"Dragoon","state":"IL","_is_real":true} +{"index":{"_id":"177"}} +{"account_number":177,"balance":48972,"firstname":"Harris","lastname":"Gross","age":40,"gender":"F","address":"468 Suydam Street","employer":"Kidstock","email":"harrisgross@kidstock.com","city":"Yettem","state":"KY","_is_real":true} +{"index":{"_id":"184"}} +{"account_number":184,"balance":9157,"firstname":"Cathy","lastname":"Morrison","age":27,"gender":"M","address":"882 Pine Street","employer":"Zytrek","email":"cathymorrison@zytrek.com","city":"Fedora","state":"FL","_is_real":true} +{"index":{"_id":"189"}} +{"account_number":189,"balance":20167,"firstname":"Ada","lastname":"Cortez","age":38,"gender":"F","address":"700 Forest Place","employer":"Micronaut","email":"adacortez@micronaut.com","city":"Eagletown","state":"TX","_is_real":true} +{"index":{"_id":"191"}} +{"account_number":191,"balance":26172,"firstname":"Barr","lastname":"Sharpe","age":28,"gender":"M","address":"428 Auburn Place","employer":"Ziggles","email":"barrsharpe@ziggles.com","city":"Springdale","state":"KS","_is_real":true} +{"index":{"_id":"196"}} +{"account_number":196,"balance":29931,"firstname":"Caldwell","lastname":"Daniel","age":28,"gender":"F","address":"405 Oliver Street","employer":"Furnigeer","email":"caldwelldaniel@furnigeer.com","city":"Zortman","state":"NE","_is_real":true} +{"index":{"_id":"204"}} +{"account_number":204,"balance":27714,"firstname":"Mavis","lastname":"Deleon","age":39,"gender":"F","address":"400 Waldane Court","employer":"Lotron","email":"mavisdeleon@lotron.com","city":"Stollings","state":"LA","_is_real":true} +{"index":{"_id":"209"}} +{"account_number":209,"balance":31052,"firstname":"Myers","lastname":"Noel","age":30,"gender":"F","address":"691 Alton Place","employer":"Greeker","email":"myersnoel@greeker.com","city":"Hinsdale","state":"KY","_is_real":true} +{"index":{"_id":"211"}} +{"account_number":211,"balance":21539,"firstname":"Graciela","lastname":"Vaughan","age":22,"gender":"M","address":"558 Montauk Court","employer":"Fishland","email":"gracielavaughan@fishland.com","city":"Madrid","state":"PA","_is_real":true} +{"index":{"_id":"216"}} +{"account_number":216,"balance":11422,"firstname":"Price","lastname":"Haley","age":35,"gender":"M","address":"233 Portland Avenue","employer":"Zeam","email":"pricehaley@zeam.com","city":"Titanic","state":"UT","_is_real":true} +{"index":{"_id":"223"}} +{"account_number":223,"balance":9528,"firstname":"Newton","lastname":"Fletcher","age":26,"gender":"F","address":"654 Dewitt Avenue","employer":"Assistia","email":"newtonfletcher@assistia.com","city":"Nipinnawasee","state":"AK","_is_real":true} +{"index":{"_id":"228"}} +{"account_number":228,"balance":10543,"firstname":"Rosella","lastname":"Albert","age":20,"gender":"M","address":"185 Gotham Avenue","employer":"Isoplex","email":"rosellaalbert@isoplex.com","city":"Finzel","state":"NY","_is_real":true} +{"index":{"_id":"230"}} +{"account_number":230,"balance":10829,"firstname":"Chris","lastname":"Raymond","age":28,"gender":"F","address":"464 Remsen Street","employer":"Cogentry","email":"chrisraymond@cogentry.com","city":"Bowmansville","state":"SD","_is_real":true} +{"index":{"_id":"235"}} +{"account_number":235,"balance":17729,"firstname":"Mcpherson","lastname":"Mueller","age":31,"gender":"M","address":"541 Strong Place","employer":"Tingles","email":"mcphersonmueller@tingles.com","city":"Brantleyville","state":"AR","_is_real":true} +{"index":{"_id":"242"}} +{"account_number":242,"balance":42318,"firstname":"Berger","lastname":"Roach","age":21,"gender":"M","address":"125 Wakeman Place","employer":"Ovium","email":"bergerroach@ovium.com","city":"Hessville","state":"WI","_is_real":true} +{"index":{"_id":"247"}} +{"account_number":247,"balance":45123,"firstname":"Mccormick","lastname":"Moon","age":37,"gender":"M","address":"582 Brighton Avenue","employer":"Norsup","email":"mccormickmoon@norsup.com","city":"Forestburg","state":"DE","_is_real":true} +{"index":{"_id":"254"}} +{"account_number":254,"balance":35104,"firstname":"Yang","lastname":"Dodson","age":21,"gender":"M","address":"531 Lott Street","employer":"Mondicil","email":"yangdodson@mondicil.com","city":"Enoree","state":"UT","_is_real":true} +{"index":{"_id":"259"}} +{"account_number":259,"balance":41877,"firstname":"Eleanor","lastname":"Gonzalez","age":30,"gender":"M","address":"800 Sumpter Street","employer":"Futuris","email":"eleanorgonzalez@futuris.com","city":"Jenkinsville","state":"ID","_is_real":true} +{"index":{"_id":"261"}} +{"account_number":261,"balance":39998,"firstname":"Millicent","lastname":"Pickett","age":34,"gender":"F","address":"722 Montieth Street","employer":"Gushkool","email":"millicentpickett@gushkool.com","city":"Norwood","state":"MS","_is_real":true} +{"index":{"_id":"266"}} +{"account_number":266,"balance":2777,"firstname":"Monique","lastname":"Conner","age":35,"gender":"F","address":"489 Metrotech Courtr","employer":"Flotonic","email":"moniqueconner@flotonic.com","city":"Retsof","state":"MD","_is_real":true} +{"index":{"_id":"273"}} +{"account_number":273,"balance":11181,"firstname":"Murphy","lastname":"Chandler","age":20,"gender":"F","address":"569 Bradford Street","employer":"Zilch","email":"murphychandler@zilch.com","city":"Vicksburg","state":"FL","_is_real":true} +{"index":{"_id":"278"}} +{"account_number":278,"balance":22530,"firstname":"Tamra","lastname":"Navarro","age":27,"gender":"F","address":"175 Woodruff Avenue","employer":"Norsul","email":"tamranavarro@norsul.com","city":"Glasgow","state":"VT","_is_real":true} +{"index":{"_id":"280"}} +{"account_number":280,"balance":3380,"firstname":"Vilma","lastname":"Shields","age":26,"gender":"F","address":"133 Berriman Street","employer":"Applidec","email":"vilmashields@applidec.com","city":"Adamstown","state":"ME","_is_real":true} +{"index":{"_id":"285"}} +{"account_number":285,"balance":47369,"firstname":"Hilda","lastname":"Phillips","age":28,"gender":"F","address":"618 Nixon Court","employer":"Comcur","email":"hildaphillips@comcur.com","city":"Siglerville","state":"NC","_is_real":true} +{"index":{"_id":"292"}} +{"account_number":292,"balance":26679,"firstname":"Morrow","lastname":"Greene","age":20,"gender":"F","address":"691 Nassau Street","employer":"Columella","email":"morrowgreene@columella.com","city":"Sanborn","state":"FL","_is_real":true} +{"index":{"_id":"297"}} +{"account_number":297,"balance":20508,"firstname":"Tucker","lastname":"Patrick","age":35,"gender":"F","address":"978 Whitwell Place","employer":"Valreda","email":"tuckerpatrick@valreda.com","city":"Deseret","state":"CO","_is_real":true} +{"index":{"_id":"300"}} +{"account_number":300,"balance":25654,"firstname":"Lane","lastname":"Tate","age":26,"gender":"F","address":"632 Kay Court","employer":"Genesynk","email":"lanetate@genesynk.com","city":"Lowell","state":"MO","_is_real":true} +{"index":{"_id":"305"}} +{"account_number":305,"balance":11655,"firstname":"Augusta","lastname":"Winters","age":29,"gender":"F","address":"377 Paerdegat Avenue","employer":"Vendblend","email":"augustawinters@vendblend.com","city":"Gwynn","state":"MA","_is_real":true} +{"index":{"_id":"312"}} +{"account_number":312,"balance":8511,"firstname":"Burgess","lastname":"Gentry","age":25,"gender":"F","address":"382 Bergen Court","employer":"Orbixtar","email":"burgessgentry@orbixtar.com","city":"Conestoga","state":"WI","_is_real":true} +{"index":{"_id":"317"}} +{"account_number":317,"balance":31968,"firstname":"Ruiz","lastname":"Morris","age":31,"gender":"F","address":"972 Dean Street","employer":"Apex","email":"ruizmorris@apex.com","city":"Jacksonwald","state":"WV","_is_real":true} +{"index":{"_id":"324"}} +{"account_number":324,"balance":44976,"firstname":"Gladys","lastname":"Erickson","age":22,"gender":"M","address":"250 Battery Avenue","employer":"Eternis","email":"gladyserickson@eternis.com","city":"Marne","state":"IA","_is_real":true} +{"index":{"_id":"329"}} +{"account_number":329,"balance":31138,"firstname":"Nellie","lastname":"Mercer","age":25,"gender":"M","address":"967 Ebony Court","employer":"Scenty","email":"nelliemercer@scenty.com","city":"Jardine","state":"AK","_is_real":true} +{"index":{"_id":"331"}} +{"account_number":331,"balance":46004,"firstname":"Gibson","lastname":"Potts","age":34,"gender":"F","address":"994 Dahill Road","employer":"Zensus","email":"gibsonpotts@zensus.com","city":"Frizzleburg","state":"CO","_is_real":true} +{"index":{"_id":"336"}} +{"account_number":336,"balance":40891,"firstname":"Dudley","lastname":"Avery","age":25,"gender":"M","address":"405 Powers Street","employer":"Genmom","email":"dudleyavery@genmom.com","city":"Clarksburg","state":"CO","_is_real":true} +{"index":{"_id":"343"}} +{"account_number":343,"balance":37684,"firstname":"Robbie","lastname":"Logan","age":29,"gender":"M","address":"488 Linden Boulevard","employer":"Hydrocom","email":"robbielogan@hydrocom.com","city":"Stockdale","state":"TN","_is_real":true} +{"index":{"_id":"348"}} +{"account_number":348,"balance":1360,"firstname":"Karina","lastname":"Russell","age":37,"gender":"M","address":"797 Moffat Street","employer":"Limozen","email":"karinarussell@limozen.com","city":"Riegelwood","state":"RI","_is_real":true} +{"index":{"_id":"350"}} +{"account_number":350,"balance":4267,"firstname":"Wyatt","lastname":"Wise","age":22,"gender":"F","address":"896 Bleecker Street","employer":"Rockyard","email":"wyattwise@rockyard.com","city":"Joes","state":"MS","_is_real":true} +{"index":{"_id":"355"}} +{"account_number":355,"balance":40961,"firstname":"Gregory","lastname":"Delacruz","age":38,"gender":"M","address":"876 Cortelyou Road","employer":"Oulu","email":"gregorydelacruz@oulu.com","city":"Waterloo","state":"WV","_is_real":true} +{"index":{"_id":"362"}} +{"account_number":362,"balance":14938,"firstname":"Jimmie","lastname":"Dejesus","age":26,"gender":"M","address":"351 Navy Walk","employer":"Ecolight","email":"jimmiedejesus@ecolight.com","city":"Berlin","state":"ME","_is_real":true} +{"index":{"_id":"367"}} +{"account_number":367,"balance":40458,"firstname":"Elaine","lastname":"Workman","age":20,"gender":"M","address":"188 Ridge Boulevard","employer":"Colaire","email":"elaineworkman@colaire.com","city":"Herbster","state":"AK","_is_real":true} +{"index":{"_id":"374"}} +{"account_number":374,"balance":19521,"firstname":"Blanchard","lastname":"Stein","age":30,"gender":"M","address":"313 Bartlett Street","employer":"Cujo","email":"blanchardstein@cujo.com","city":"Cascades","state":"OR","_is_real":true} +{"index":{"_id":"379"}} +{"account_number":379,"balance":12962,"firstname":"Ruthie","lastname":"Lamb","age":21,"gender":"M","address":"796 Rockaway Avenue","employer":"Incubus","email":"ruthielamb@incubus.com","city":"Hickory","state":"TX","_is_real":true} +{"index":{"_id":"381"}} +{"account_number":381,"balance":40978,"firstname":"Sophie","lastname":"Mays","age":31,"gender":"M","address":"261 Varanda Place","employer":"Uneeq","email":"sophiemays@uneeq.com","city":"Cressey","state":"AR","_is_real":true} +{"index":{"_id":"386"}} +{"account_number":386,"balance":42588,"firstname":"Wallace","lastname":"Barr","age":39,"gender":"F","address":"246 Beverly Road","employer":"Concility","email":"wallacebarr@concility.com","city":"Durham","state":"IN","_is_real":true} +{"index":{"_id":"393"}} +{"account_number":393,"balance":43936,"firstname":"William","lastname":"Kelly","age":24,"gender":"M","address":"178 Lawrence Avenue","employer":"Techtrix","email":"williamkelly@techtrix.com","city":"Orin","state":"PA","_is_real":true} +{"index":{"_id":"398"}} +{"account_number":398,"balance":8543,"firstname":"Leticia","lastname":"Duran","age":35,"gender":"F","address":"305 Senator Street","employer":"Xleen","email":"leticiaduran@xleen.com","city":"Cavalero","state":"PA","_is_real":true} +{"index":{"_id":"401"}} +{"account_number":401,"balance":29408,"firstname":"Contreras","lastname":"Randolph","age":38,"gender":"M","address":"104 Lewis Avenue","employer":"Inrt","email":"contrerasrandolph@inrt.com","city":"Chesapeake","state":"CT","_is_real":true} +{"index":{"_id":"406"}} +{"account_number":406,"balance":28127,"firstname":"Mccarthy","lastname":"Dunlap","age":28,"gender":"F","address":"684 Seacoast Terrace","employer":"Canopoly","email":"mccarthydunlap@canopoly.com","city":"Elliott","state":"NC","_is_real":true} +{"index":{"_id":"413"}} +{"account_number":413,"balance":15631,"firstname":"Pugh","lastname":"Hamilton","age":39,"gender":"F","address":"124 Euclid Avenue","employer":"Techade","email":"pughhamilton@techade.com","city":"Beaulieu","state":"CA","_is_real":true} +{"index":{"_id":"418"}} +{"account_number":418,"balance":10207,"firstname":"Reed","lastname":"Goff","age":32,"gender":"M","address":"959 Everit Street","employer":"Zillan","email":"reedgoff@zillan.com","city":"Hiko","state":"WV","_is_real":true} +{"index":{"_id":"420"}} +{"account_number":420,"balance":44699,"firstname":"Brandie","lastname":"Hayden","age":22,"gender":"M","address":"291 Ash Street","employer":"Digifad","email":"brandiehayden@digifad.com","city":"Spelter","state":"NM","_is_real":true} +{"index":{"_id":"425"}} +{"account_number":425,"balance":41308,"firstname":"Queen","lastname":"Leach","age":30,"gender":"M","address":"105 Fair Street","employer":"Magneato","email":"queenleach@magneato.com","city":"Barronett","state":"NH","_is_real":true} +{"index":{"_id":"432"}} +{"account_number":432,"balance":28969,"firstname":"Preston","lastname":"Ferguson","age":40,"gender":"F","address":"239 Greenwood Avenue","employer":"Bitendrex","email":"prestonferguson@bitendrex.com","city":"Idledale","state":"ND","_is_real":true} +{"index":{"_id":"437"}} +{"account_number":437,"balance":41225,"firstname":"Rosales","lastname":"Marquez","age":29,"gender":"M","address":"873 Ryerson Street","employer":"Ronelon","email":"rosalesmarquez@ronelon.com","city":"Allendale","state":"CA","_is_real":true} +{"index":{"_id":"444"}} +{"account_number":444,"balance":44219,"firstname":"Dolly","lastname":"Finch","age":24,"gender":"F","address":"974 Interborough Parkway","employer":"Zytrac","email":"dollyfinch@zytrac.com","city":"Vowinckel","state":"WY","_is_real":true} +{"index":{"_id":"449"}} +{"account_number":449,"balance":41950,"firstname":"Barnett","lastname":"Cantrell","age":39,"gender":"F","address":"945 Bedell Lane","employer":"Zentility","email":"barnettcantrell@zentility.com","city":"Swartzville","state":"ND","_is_real":true} +{"index":{"_id":"451"}} +{"account_number":451,"balance":31950,"firstname":"Mason","lastname":"Mcleod","age":31,"gender":"F","address":"438 Havemeyer Street","employer":"Omatom","email":"masonmcleod@omatom.com","city":"Ryderwood","state":"NE","_is_real":true} +{"index":{"_id":"456"}} +{"account_number":456,"balance":21419,"firstname":"Solis","lastname":"Kline","age":33,"gender":"M","address":"818 Ashford Street","employer":"Vetron","email":"soliskline@vetron.com","city":"Ruffin","state":"NY","_is_real":true} +{"index":{"_id":"463"}} +{"account_number":463,"balance":36672,"firstname":"Heidi","lastname":"Acosta","age":20,"gender":"F","address":"692 Kenmore Terrace","employer":"Elpro","email":"heidiacosta@elpro.com","city":"Ezel","state":"SD","_is_real":true} +{"index":{"_id":"468"}} +{"account_number":468,"balance":18400,"firstname":"Foreman","lastname":"Fowler","age":40,"gender":"M","address":"443 Jackson Court","employer":"Zillactic","email":"foremanfowler@zillactic.com","city":"Wakarusa","state":"WA","_is_real":true} +{"index":{"_id":"470"}} +{"account_number":470,"balance":20455,"firstname":"Schneider","lastname":"Hull","age":35,"gender":"M","address":"724 Apollo Street","employer":"Exospeed","email":"schneiderhull@exospeed.com","city":"Watchtower","state":"ID","_is_real":true} +{"index":{"_id":"475"}} +{"account_number":475,"balance":24427,"firstname":"Morales","lastname":"Jacobs","age":22,"gender":"F","address":"225 Desmond Court","employer":"Oronoko","email":"moralesjacobs@oronoko.com","city":"Clayville","state":"CT","_is_real":true} +{"index":{"_id":"482"}} +{"account_number":482,"balance":14834,"firstname":"Janie","lastname":"Bass","age":39,"gender":"M","address":"781 Grattan Street","employer":"Manglo","email":"janiebass@manglo.com","city":"Kenwood","state":"IA","_is_real":true} +{"index":{"_id":"487"}} +{"account_number":487,"balance":30718,"firstname":"Sawyer","lastname":"Vincent","age":26,"gender":"F","address":"238 Lancaster Avenue","employer":"Brainquil","email":"sawyervincent@brainquil.com","city":"Galesville","state":"MS","_is_real":true} +{"index":{"_id":"494"}} +{"account_number":494,"balance":3592,"firstname":"Holden","lastname":"Bowen","age":30,"gender":"M","address":"374 Elmwood Avenue","employer":"Endipine","email":"holdenbowen@endipine.com","city":"Rosine","state":"ID","_is_real":true} +{"index":{"_id":"499"}} +{"account_number":499,"balance":26060,"firstname":"Lara","lastname":"Perkins","age":26,"gender":"M","address":"703 Monroe Street","employer":"Paprikut","email":"laraperkins@paprikut.com","city":"Barstow","state":"NY","_is_real":true} +{"index":{"_id":"502"}} +{"account_number":502,"balance":31898,"firstname":"Woodard","lastname":"Bailey","age":31,"gender":"F","address":"585 Albee Square","employer":"Imperium","email":"woodardbailey@imperium.com","city":"Matheny","state":"MT","_is_real":true} +{"index":{"_id":"507"}} +{"account_number":507,"balance":27675,"firstname":"Blankenship","lastname":"Ramirez","age":31,"gender":"M","address":"630 Graham Avenue","employer":"Bytrex","email":"blankenshipramirez@bytrex.com","city":"Bancroft","state":"CT","_is_real":true} +{"index":{"_id":"514"}} +{"account_number":514,"balance":30125,"firstname":"Solomon","lastname":"Bush","age":34,"gender":"M","address":"409 Harkness Avenue","employer":"Snacktion","email":"solomonbush@snacktion.com","city":"Grayhawk","state":"TX","_is_real":true} +{"index":{"_id":"519"}} +{"account_number":519,"balance":3282,"firstname":"Lorna","lastname":"Franco","age":31,"gender":"F","address":"722 Schenck Court","employer":"Zentia","email":"lornafranco@zentia.com","city":"National","state":"FL","_is_real":true} +{"index":{"_id":"521"}} +{"account_number":521,"balance":16348,"firstname":"Josefa","lastname":"Buckley","age":34,"gender":"F","address":"848 Taylor Street","employer":"Mazuda","email":"josefabuckley@mazuda.com","city":"Saranap","state":"NM","_is_real":true} +{"index":{"_id":"526"}} +{"account_number":526,"balance":35375,"firstname":"Sweeney","lastname":"Fulton","age":33,"gender":"F","address":"550 Martense Street","employer":"Cormoran","email":"sweeneyfulton@cormoran.com","city":"Chalfant","state":"IA","_is_real":true} +{"index":{"_id":"533"}} +{"account_number":533,"balance":13761,"firstname":"Margarita","lastname":"Diaz","age":23,"gender":"M","address":"295 Tapscott Street","employer":"Zilodyne","email":"margaritadiaz@zilodyne.com","city":"Hondah","state":"ID","_is_real":true} +{"index":{"_id":"538"}} +{"account_number":538,"balance":16416,"firstname":"Koch","lastname":"Barker","age":21,"gender":"M","address":"919 Gerry Street","employer":"Xplor","email":"kochbarker@xplor.com","city":"Dixie","state":"WY","_is_real":true} +{"index":{"_id":"540"}} +{"account_number":540,"balance":40235,"firstname":"Tammy","lastname":"Wiggins","age":32,"gender":"F","address":"186 Schenectady Avenue","employer":"Speedbolt","email":"tammywiggins@speedbolt.com","city":"Salvo","state":"LA","_is_real":true} +{"index":{"_id":"545"}} +{"account_number":545,"balance":27011,"firstname":"Lena","lastname":"Lucas","age":20,"gender":"M","address":"110 Lamont Court","employer":"Kindaloo","email":"lenalucas@kindaloo.com","city":"Harleigh","state":"KY","_is_real":true} +{"index":{"_id":"552"}} +{"account_number":552,"balance":14727,"firstname":"Kate","lastname":"Estes","age":39,"gender":"M","address":"785 Willmohr Street","employer":"Rodeocean","email":"kateestes@rodeocean.com","city":"Elfrida","state":"HI","_is_real":true} +{"index":{"_id":"557"}} +{"account_number":557,"balance":3119,"firstname":"Landry","lastname":"Buck","age":20,"gender":"M","address":"558 Schweikerts Walk","employer":"Protodyne","email":"landrybuck@protodyne.com","city":"Edneyville","state":"AL","_is_real":true} +{"index":{"_id":"564"}} +{"account_number":564,"balance":43631,"firstname":"Owens","lastname":"Bowers","age":22,"gender":"M","address":"842 Congress Street","employer":"Nspire","email":"owensbowers@nspire.com","city":"Machias","state":"VA","_is_real":true} +{"index":{"_id":"569"}} +{"account_number":569,"balance":40019,"firstname":"Sherri","lastname":"Rowe","age":39,"gender":"F","address":"591 Arlington Place","employer":"Netility","email":"sherrirowe@netility.com","city":"Bridgetown","state":"SC","_is_real":true} +{"index":{"_id":"571"}} +{"account_number":571,"balance":3014,"firstname":"Ayers","lastname":"Duffy","age":28,"gender":"F","address":"721 Wortman Avenue","employer":"Aquasseur","email":"ayersduffy@aquasseur.com","city":"Tilleda","state":"MS","_is_real":true} +{"index":{"_id":"576"}} +{"account_number":576,"balance":29682,"firstname":"Helena","lastname":"Robertson","age":33,"gender":"F","address":"774 Devon Avenue","employer":"Vicon","email":"helenarobertson@vicon.com","city":"Dyckesville","state":"NV","_is_real":true} +{"index":{"_id":"583"}} +{"account_number":583,"balance":26558,"firstname":"Castro","lastname":"West","age":34,"gender":"F","address":"814 Williams Avenue","employer":"Cipromox","email":"castrowest@cipromox.com","city":"Nescatunga","state":"IL","_is_real":true} +{"index":{"_id":"588"}} +{"account_number":588,"balance":43531,"firstname":"Martina","lastname":"Collins","age":31,"gender":"M","address":"301 Anna Court","employer":"Geekwagon","email":"martinacollins@geekwagon.com","city":"Oneida","state":"VA","_is_real":true} +{"index":{"_id":"590"}} +{"account_number":590,"balance":4652,"firstname":"Ladonna","lastname":"Tucker","age":31,"gender":"F","address":"162 Kane Place","employer":"Infotrips","email":"ladonnatucker@infotrips.com","city":"Utting","state":"IA","_is_real":true} +{"index":{"_id":"595"}} +{"account_number":595,"balance":12478,"firstname":"Mccall","lastname":"Britt","age":36,"gender":"F","address":"823 Hill Street","employer":"Cablam","email":"mccallbritt@cablam.com","city":"Vernon","state":"CA","_is_real":true} +{"index":{"_id":"603"}} +{"account_number":603,"balance":28145,"firstname":"Janette","lastname":"Guzman","age":31,"gender":"F","address":"976 Kingston Avenue","employer":"Splinx","email":"janetteguzman@splinx.com","city":"Boomer","state":"NC","_is_real":true} +{"index":{"_id":"608"}} +{"account_number":608,"balance":47091,"firstname":"Carey","lastname":"Whitley","age":32,"gender":"F","address":"976 Lawrence Street","employer":"Poshome","email":"careywhitley@poshome.com","city":"Weogufka","state":"NE","_is_real":true} +{"index":{"_id":"610"}} +{"account_number":610,"balance":40571,"firstname":"Foster","lastname":"Weber","age":24,"gender":"F","address":"323 Rochester Avenue","employer":"Firewax","email":"fosterweber@firewax.com","city":"Winston","state":"NY","_is_real":true} +{"index":{"_id":"615"}} +{"account_number":615,"balance":28726,"firstname":"Delgado","lastname":"Curry","age":28,"gender":"F","address":"706 Butler Street","employer":"Zoxy","email":"delgadocurry@zoxy.com","city":"Gracey","state":"SD","_is_real":true} +{"index":{"_id":"622"}} +{"account_number":622,"balance":9661,"firstname":"Paulette","lastname":"Hartman","age":38,"gender":"M","address":"375 Emerald Street","employer":"Locazone","email":"paulettehartman@locazone.com","city":"Canterwood","state":"OH","_is_real":true} +{"index":{"_id":"627"}} +{"account_number":627,"balance":47546,"firstname":"Crawford","lastname":"Sears","age":37,"gender":"F","address":"686 Eastern Parkway","employer":"Updat","email":"crawfordsears@updat.com","city":"Bison","state":"VT","_is_real":true} +{"index":{"_id":"634"}} +{"account_number":634,"balance":29805,"firstname":"Deloris","lastname":"Levy","age":38,"gender":"M","address":"838 Foster Avenue","employer":"Homelux","email":"delorislevy@homelux.com","city":"Kempton","state":"PA","_is_real":true} +{"index":{"_id":"639"}} +{"account_number":639,"balance":28875,"firstname":"Caitlin","lastname":"Clements","age":32,"gender":"F","address":"627 Aster Court","employer":"Bunga","email":"caitlinclements@bunga.com","city":"Cetronia","state":"SC","_is_real":true} +{"index":{"_id":"641"}} +{"account_number":641,"balance":18345,"firstname":"Sheppard","lastname":"Everett","age":39,"gender":"F","address":"791 Norwood Avenue","employer":"Roboid","email":"sheppardeverett@roboid.com","city":"Selma","state":"AK","_is_real":true} +{"index":{"_id":"646"}} +{"account_number":646,"balance":15559,"firstname":"Lavonne","lastname":"Reyes","age":31,"gender":"F","address":"983 Newport Street","employer":"Parcoe","email":"lavonnereyes@parcoe.com","city":"Monument","state":"LA","_is_real":true} +{"index":{"_id":"653"}} +{"account_number":653,"balance":7606,"firstname":"Marcia","lastname":"Bennett","age":33,"gender":"F","address":"455 Bragg Street","employer":"Opticall","email":"marciabennett@opticall.com","city":"Magnolia","state":"NC","_is_real":true} +{"index":{"_id":"658"}} +{"account_number":658,"balance":10210,"firstname":"Bass","lastname":"Mcconnell","age":32,"gender":"F","address":"274 Ocean Avenue","employer":"Combot","email":"bassmcconnell@combot.com","city":"Beyerville","state":"OH","_is_real":true} +{"index":{"_id":"660"}} +{"account_number":660,"balance":46427,"firstname":"Moon","lastname":"Wood","age":33,"gender":"F","address":"916 Amersfort Place","employer":"Olucore","email":"moonwood@olucore.com","city":"Como","state":"VA","_is_real":true} +{"index":{"_id":"665"}} +{"account_number":665,"balance":15215,"firstname":"Britney","lastname":"Young","age":36,"gender":"M","address":"766 Sackman Street","employer":"Geoforma","email":"britneyyoung@geoforma.com","city":"Tuttle","state":"WI","_is_real":true} +{"index":{"_id":"672"}} +{"account_number":672,"balance":12621,"firstname":"Camille","lastname":"Munoz","age":36,"gender":"F","address":"959 Lewis Place","employer":"Vantage","email":"camillemunoz@vantage.com","city":"Whitmer","state":"IN","_is_real":true} +{"index":{"_id":"677"}} +{"account_number":677,"balance":8491,"firstname":"Snider","lastname":"Benton","age":26,"gender":"M","address":"827 Evans Street","employer":"Medicroix","email":"sniderbenton@medicroix.com","city":"Kaka","state":"UT","_is_real":true} +{"index":{"_id":"684"}} +{"account_number":684,"balance":46091,"firstname":"Warren","lastname":"Snow","age":25,"gender":"M","address":"756 Oakland Place","employer":"Bizmatic","email":"warrensnow@bizmatic.com","city":"Hatteras","state":"NE","_is_real":true} +{"index":{"_id":"689"}} +{"account_number":689,"balance":14985,"firstname":"Ines","lastname":"Chaney","age":28,"gender":"M","address":"137 Dikeman Street","employer":"Zidant","email":"ineschaney@zidant.com","city":"Nettie","state":"DC","_is_real":true} +{"index":{"_id":"691"}} +{"account_number":691,"balance":10792,"firstname":"Mclean","lastname":"Colon","age":22,"gender":"M","address":"876 Classon Avenue","employer":"Elentrix","email":"mcleancolon@elentrix.com","city":"Unionville","state":"OK","_is_real":true} +{"index":{"_id":"696"}} +{"account_number":696,"balance":17568,"firstname":"Crane","lastname":"Matthews","age":32,"gender":"F","address":"721 Gerritsen Avenue","employer":"Intradisk","email":"cranematthews@intradisk.com","city":"Brewster","state":"WV","_is_real":true} +{"index":{"_id":"704"}} +{"account_number":704,"balance":45347,"firstname":"Peters","lastname":"Kent","age":22,"gender":"F","address":"871 Independence Avenue","employer":"Extragen","email":"peterskent@extragen.com","city":"Morriston","state":"CA","_is_real":true} +{"index":{"_id":"709"}} +{"account_number":709,"balance":11015,"firstname":"Abbott","lastname":"Odom","age":29,"gender":"M","address":"893 Union Street","employer":"Jimbies","email":"abbottodom@jimbies.com","city":"Leeper","state":"NJ","_is_real":true} +{"index":{"_id":"711"}} +{"account_number":711,"balance":26939,"firstname":"Villarreal","lastname":"Horton","age":35,"gender":"F","address":"861 Creamer Street","employer":"Lexicondo","email":"villarrealhorton@lexicondo.com","city":"Lydia","state":"MS","_is_real":true} +{"index":{"_id":"716"}} +{"account_number":716,"balance":19789,"firstname":"Paul","lastname":"Mason","age":34,"gender":"F","address":"618 Nichols Avenue","employer":"Slax","email":"paulmason@slax.com","city":"Snowville","state":"OK","_is_real":true} +{"index":{"_id":"723"}} +{"account_number":723,"balance":16421,"firstname":"Nixon","lastname":"Moran","age":27,"gender":"M","address":"569 Campus Place","employer":"Cuizine","email":"nixonmoran@cuizine.com","city":"Buxton","state":"DC","_is_real":true} +{"index":{"_id":"728"}} +{"account_number":728,"balance":44818,"firstname":"Conley","lastname":"Preston","age":28,"gender":"M","address":"450 Coventry Road","employer":"Obones","email":"conleypreston@obones.com","city":"Alden","state":"CO","_is_real":true} +{"index":{"_id":"730"}} +{"account_number":730,"balance":41299,"firstname":"Moore","lastname":"Lee","age":30,"gender":"M","address":"797 Turner Place","employer":"Orbean","email":"moorelee@orbean.com","city":"Highland","state":"DE","_is_real":true} +{"index":{"_id":"735"}} +{"account_number":735,"balance":3984,"firstname":"Loraine","lastname":"Willis","age":32,"gender":"F","address":"928 Grove Street","employer":"Gadtron","email":"lorainewillis@gadtron.com","city":"Lowgap","state":"NY","_is_real":true} +{"index":{"_id":"742"}} +{"account_number":742,"balance":24765,"firstname":"Merle","lastname":"Wooten","age":26,"gender":"M","address":"317 Pooles Lane","employer":"Tropolis","email":"merlewooten@tropolis.com","city":"Bentley","state":"ND","_is_real":true} +{"index":{"_id":"747"}} +{"account_number":747,"balance":16617,"firstname":"Diaz","lastname":"Austin","age":38,"gender":"M","address":"676 Harway Avenue","employer":"Irack","email":"diazaustin@irack.com","city":"Cliff","state":"HI","_is_real":true} +{"index":{"_id":"754"}} +{"account_number":754,"balance":10779,"firstname":"Jones","lastname":"Vega","age":25,"gender":"F","address":"795 India Street","employer":"Gluid","email":"jonesvega@gluid.com","city":"Tyhee","state":"FL","_is_real":true} +{"index":{"_id":"759"}} +{"account_number":759,"balance":38007,"firstname":"Rose","lastname":"Carlson","age":27,"gender":"M","address":"987 Navy Street","employer":"Aquasure","email":"rosecarlson@aquasure.com","city":"Carlton","state":"CT","_is_real":true} +{"index":{"_id":"761"}} +{"account_number":761,"balance":7663,"firstname":"Rae","lastname":"Juarez","age":34,"gender":"F","address":"560 Gilmore Court","employer":"Entropix","email":"raejuarez@entropix.com","city":"Northchase","state":"ID","_is_real":true} +{"index":{"_id":"766"}} +{"account_number":766,"balance":21957,"firstname":"Thomas","lastname":"Gillespie","age":38,"gender":"M","address":"993 Williams Place","employer":"Octocore","email":"thomasgillespie@octocore.com","city":"Defiance","state":"MS","_is_real":true} +{"index":{"_id":"773"}} +{"account_number":773,"balance":31126,"firstname":"Liza","lastname":"Coffey","age":36,"gender":"F","address":"540 Bulwer Place","employer":"Assurity","email":"lizacoffey@assurity.com","city":"Gilgo","state":"WV","_is_real":true} +{"index":{"_id":"778"}} +{"account_number":778,"balance":46007,"firstname":"Underwood","lastname":"Wheeler","age":28,"gender":"M","address":"477 Provost Street","employer":"Decratex","email":"underwoodwheeler@decratex.com","city":"Sardis","state":"ID","_is_real":true} +{"index":{"_id":"780"}} +{"account_number":780,"balance":4682,"firstname":"Maryanne","lastname":"Hendricks","age":26,"gender":"F","address":"709 Wolcott Street","employer":"Sarasonic","email":"maryannehendricks@sarasonic.com","city":"Santel","state":"NH","_is_real":true} +{"index":{"_id":"785"}} +{"account_number":785,"balance":25078,"firstname":"Fields","lastname":"Lester","age":29,"gender":"M","address":"808 Chestnut Avenue","employer":"Visualix","email":"fieldslester@visualix.com","city":"Rowe","state":"PA","_is_real":true} +{"index":{"_id":"792"}} +{"account_number":792,"balance":13109,"firstname":"Becky","lastname":"Jimenez","age":40,"gender":"F","address":"539 Front Street","employer":"Isologia","email":"beckyjimenez@isologia.com","city":"Summertown","state":"MI","_is_real":true} +{"index":{"_id":"797"}} +{"account_number":797,"balance":6854,"firstname":"Lindsay","lastname":"Mills","age":26,"gender":"F","address":"919 Quay Street","employer":"Zoinage","email":"lindsaymills@zoinage.com","city":"Elliston","state":"VA","_is_real":true} +{"index":{"_id":"800"}} +{"account_number":800,"balance":26217,"firstname":"Candy","lastname":"Oconnor","age":28,"gender":"M","address":"200 Newel Street","employer":"Radiantix","email":"candyoconnor@radiantix.com","city":"Sandston","state":"OH","_is_real":true} +{"index":{"_id":"805"}} +{"account_number":805,"balance":18426,"firstname":"Jackson","lastname":"Sampson","age":27,"gender":"F","address":"722 Kenmore Court","employer":"Daido","email":"jacksonsampson@daido.com","city":"Bellamy","state":"ME","_is_real":true} +{"index":{"_id":"812"}} +{"account_number":812,"balance":42593,"firstname":"Graves","lastname":"Newman","age":32,"gender":"F","address":"916 Joralemon Street","employer":"Ecrater","email":"gravesnewman@ecrater.com","city":"Crown","state":"PA","_is_real":true} +{"index":{"_id":"817"}} +{"account_number":817,"balance":36582,"firstname":"Padilla","lastname":"Bauer","age":36,"gender":"F","address":"310 Cadman Plaza","employer":"Exoblue","email":"padillabauer@exoblue.com","city":"Ahwahnee","state":"MN","_is_real":true} +{"index":{"_id":"824"}} +{"account_number":824,"balance":6053,"firstname":"Dyer","lastname":"Henson","age":33,"gender":"M","address":"650 Seaview Avenue","employer":"Nitracyr","email":"dyerhenson@nitracyr.com","city":"Gibsonia","state":"KS","_is_real":true} +{"index":{"_id":"829"}} +{"account_number":829,"balance":20263,"firstname":"Althea","lastname":"Bell","age":37,"gender":"M","address":"319 Cook Street","employer":"Hyplex","email":"altheabell@hyplex.com","city":"Wadsworth","state":"DC","_is_real":true} +{"index":{"_id":"831"}} +{"account_number":831,"balance":25375,"firstname":"Wendy","lastname":"Savage","age":37,"gender":"M","address":"421 Veranda Place","employer":"Neurocell","email":"wendysavage@neurocell.com","city":"Fresno","state":"MS","_is_real":true} +{"index":{"_id":"836"}} +{"account_number":836,"balance":20797,"firstname":"Lloyd","lastname":"Lindsay","age":25,"gender":"F","address":"953 Dinsmore Place","employer":"Suretech","email":"lloydlindsay@suretech.com","city":"Conway","state":"VA","_is_real":true} +{"index":{"_id":"843"}} +{"account_number":843,"balance":15555,"firstname":"Patricia","lastname":"Barton","age":34,"gender":"F","address":"406 Seabring Street","employer":"Providco","email":"patriciabarton@providco.com","city":"Avoca","state":"RI","_is_real":true} +{"index":{"_id":"848"}} +{"account_number":848,"balance":15443,"firstname":"Carmella","lastname":"Cash","age":38,"gender":"M","address":"988 Exeter Street","employer":"Bristo","email":"carmellacash@bristo.com","city":"Northridge","state":"ID","_is_real":true} +{"index":{"_id":"850"}} +{"account_number":850,"balance":6531,"firstname":"Carlene","lastname":"Gaines","age":37,"gender":"F","address":"753 Monroe Place","employer":"Naxdis","email":"carlenegaines@naxdis.com","city":"Genoa","state":"OR","_is_real":true} +{"index":{"_id":"855"}} +{"account_number":855,"balance":40170,"firstname":"Mia","lastname":"Stevens","age":31,"gender":"F","address":"326 Driggs Avenue","employer":"Aeora","email":"miastevens@aeora.com","city":"Delwood","state":"IL","_is_real":true} +{"index":{"_id":"862"}} +{"account_number":862,"balance":38792,"firstname":"Clayton","lastname":"Golden","age":38,"gender":"F","address":"620 Regent Place","employer":"Accusage","email":"claytongolden@accusage.com","city":"Ona","state":"NC","_is_real":true} +{"index":{"_id":"867"}} +{"account_number":867,"balance":45453,"firstname":"Blanca","lastname":"Ellison","age":23,"gender":"F","address":"593 McKibben Street","employer":"Koogle","email":"blancaellison@koogle.com","city":"Frystown","state":"WY","_is_real":true} +{"index":{"_id":"874"}} +{"account_number":874,"balance":23079,"firstname":"Lynette","lastname":"Higgins","age":22,"gender":"M","address":"377 McKinley Avenue","employer":"Menbrain","email":"lynettehiggins@menbrain.com","city":"Manitou","state":"TX","_is_real":true} +{"index":{"_id":"879"}} +{"account_number":879,"balance":48332,"firstname":"Sabrina","lastname":"Lancaster","age":31,"gender":"F","address":"382 Oak Street","employer":"Webiotic","email":"sabrinalancaster@webiotic.com","city":"Lindisfarne","state":"AZ","_is_real":true} +{"index":{"_id":"881"}} +{"account_number":881,"balance":26684,"firstname":"Barnes","lastname":"Ware","age":38,"gender":"F","address":"666 Hooper Street","employer":"Norali","email":"barnesware@norali.com","city":"Cazadero","state":"GA","_is_real":true} +{"index":{"_id":"886"}} +{"account_number":886,"balance":14867,"firstname":"Willa","lastname":"Leblanc","age":38,"gender":"F","address":"773 Bergen Street","employer":"Nurali","email":"willaleblanc@nurali.com","city":"Hilltop","state":"NC","_is_real":true} +{"index":{"_id":"893"}} +{"account_number":893,"balance":42584,"firstname":"Moses","lastname":"Campos","age":38,"gender":"F","address":"991 Bevy Court","employer":"Trollery","email":"mosescampos@trollery.com","city":"Freetown","state":"AK","_is_real":true} +{"index":{"_id":"898"}} +{"account_number":898,"balance":12019,"firstname":"Lori","lastname":"Stevenson","age":29,"gender":"M","address":"910 Coles Street","employer":"Honotron","email":"loristevenson@honotron.com","city":"Shindler","state":"VT","_is_real":true} +{"index":{"_id":"901"}} +{"account_number":901,"balance":35038,"firstname":"Irma","lastname":"Dotson","age":23,"gender":"F","address":"245 Mayfair Drive","employer":"Bleeko","email":"irmadotson@bleeko.com","city":"Lodoga","state":"UT","_is_real":true} +{"index":{"_id":"906"}} +{"account_number":906,"balance":24073,"firstname":"Vicki","lastname":"Suarez","age":36,"gender":"M","address":"829 Roosevelt Place","employer":"Utara","email":"vickisuarez@utara.com","city":"Albrightsville","state":"AR","_is_real":true} +{"index":{"_id":"913"}} +{"account_number":913,"balance":47657,"firstname":"Margery","lastname":"Monroe","age":25,"gender":"M","address":"941 Fanchon Place","employer":"Exerta","email":"margerymonroe@exerta.com","city":"Bannock","state":"MD","_is_real":true} +{"index":{"_id":"918"}} +{"account_number":918,"balance":36776,"firstname":"Dianna","lastname":"Hernandez","age":25,"gender":"M","address":"499 Moultrie Street","employer":"Isologica","email":"diannahernandez@isologica.com","city":"Falconaire","state":"ID","_is_real":true} +{"index":{"_id":"920"}} +{"account_number":920,"balance":41513,"firstname":"Jerri","lastname":"Mitchell","age":26,"gender":"M","address":"831 Kent Street","employer":"Tasmania","email":"jerrimitchell@tasmania.com","city":"Cotopaxi","state":"IA","_is_real":true} +{"index":{"_id":"925"}} +{"account_number":925,"balance":18295,"firstname":"Rosario","lastname":"Jackson","age":24,"gender":"M","address":"178 Leonora Court","employer":"Progenex","email":"rosariojackson@progenex.com","city":"Rivereno","state":"DE","_is_real":true} +{"index":{"_id":"932"}} +{"account_number":932,"balance":3111,"firstname":"Summer","lastname":"Porter","age":33,"gender":"F","address":"949 Grand Avenue","employer":"Multiflex","email":"summerporter@multiflex.com","city":"Spokane","state":"OK","_is_real":true} +{"index":{"_id":"937"}} +{"account_number":937,"balance":43491,"firstname":"Selma","lastname":"Anderson","age":24,"gender":"M","address":"205 Reed Street","employer":"Dadabase","email":"selmaanderson@dadabase.com","city":"Malo","state":"AL","_is_real":true} +{"index":{"_id":"944"}} +{"account_number":944,"balance":46478,"firstname":"Donaldson","lastname":"Woodard","age":38,"gender":"F","address":"498 Laurel Avenue","employer":"Zogak","email":"donaldsonwoodard@zogak.com","city":"Hasty","state":"ID","_is_real":true} +{"index":{"_id":"949"}} +{"account_number":949,"balance":48703,"firstname":"Latasha","lastname":"Mullins","age":29,"gender":"F","address":"272 Lefferts Place","employer":"Zenolux","email":"latashamullins@zenolux.com","city":"Kieler","state":"MN","_is_real":true} +{"index":{"_id":"951"}} +{"account_number":951,"balance":36337,"firstname":"Tran","lastname":"Burris","age":25,"gender":"F","address":"561 Rutland Road","employer":"Geoform","email":"tranburris@geoform.com","city":"Longbranch","state":"IL","_is_real":true} +{"index":{"_id":"956"}} +{"account_number":956,"balance":19477,"firstname":"Randall","lastname":"Lynch","age":22,"gender":"F","address":"490 Madison Place","employer":"Cosmetex","email":"randalllynch@cosmetex.com","city":"Wells","state":"SD","_is_real":true} +{"index":{"_id":"963"}} +{"account_number":963,"balance":30461,"firstname":"Griffin","lastname":"Sheppard","age":20,"gender":"M","address":"682 Linden Street","employer":"Zanymax","email":"griffinsheppard@zanymax.com","city":"Fannett","state":"NM","_is_real":true} +{"index":{"_id":"968"}} +{"account_number":968,"balance":32371,"firstname":"Luella","lastname":"Burch","age":39,"gender":"M","address":"684 Arkansas Drive","employer":"Krag","email":"luellaburch@krag.com","city":"Brambleton","state":"SD","_is_real":true} +{"index":{"_id":"970"}} +{"account_number":970,"balance":19648,"firstname":"Forbes","lastname":"Wallace","age":28,"gender":"M","address":"990 Mill Road","employer":"Pheast","email":"forbeswallace@pheast.com","city":"Lopezo","state":"AK","_is_real":true} +{"index":{"_id":"975"}} +{"account_number":975,"balance":5239,"firstname":"Delores","lastname":"Booker","age":27,"gender":"F","address":"328 Conselyea Street","employer":"Centice","email":"deloresbooker@centice.com","city":"Williams","state":"HI","_is_real":true} +{"index":{"_id":"982"}} +{"account_number":982,"balance":16511,"firstname":"Buck","lastname":"Robinson","age":24,"gender":"M","address":"301 Melrose Street","employer":"Calcu","email":"buckrobinson@calcu.com","city":"Welch","state":"PA","_is_real":true} +{"index":{"_id":"987"}} +{"account_number":987,"balance":4072,"firstname":"Brock","lastname":"Sandoval","age":20,"gender":"F","address":"977 Gem Street","employer":"Fiberox","email":"brocksandoval@fiberox.com","city":"Celeryville","state":"NY","_is_real":true} +{"index":{"_id":"994"}} +{"account_number":994,"balance":33298,"firstname":"Madge","lastname":"Holcomb","age":31,"gender":"M","address":"612 Hawthorne Street","employer":"Escenta","email":"madgeholcomb@escenta.com","city":"Alafaya","state":"OR","_is_real":true} +{"index":{"_id":"999"}} +{"account_number":999,"balance":6087,"firstname":"Dorothy","lastname":"Barron","age":22,"gender":"F","address":"499 Laurel Avenue","employer":"Xurban","email":"dorothybarron@xurban.com","city":"Belvoir","state":"CA","_is_real":true} +{"index":{"_id":"4"}} +{"account_number":4,"balance":27658,"firstname":"Rodriquez","lastname":"Flores","age":31,"gender":"F","address":"986 Wyckoff Avenue","employer":"Tourmania","email":"rodriquezflores@tourmania.com","city":"Eastvale","state":"HI","_is_real":true} +{"index":{"_id":"9"}} +{"account_number":9,"balance":24776,"firstname":"Opal","lastname":"Meadows","age":39,"gender":"M","address":"963 Neptune Avenue","employer":"Cedward","email":"opalmeadows@cedward.com","city":"Olney","state":"OH","_is_real":true} +{"index":{"_id":"11"}} +{"account_number":11,"balance":20203,"firstname":"Jenkins","lastname":"Haney","age":20,"gender":"M","address":"740 Ferry Place","employer":"Qimonk","email":"jenkinshaney@qimonk.com","city":"Steinhatchee","state":"GA","_is_real":true} +{"index":{"_id":"16"}} +{"account_number":16,"balance":35883,"firstname":"Adrian","lastname":"Pitts","age":34,"gender":"F","address":"963 Fay Court","employer":"Combogene","email":"adrianpitts@combogene.com","city":"Remington","state":"SD","_is_real":true} +{"index":{"_id":"23"}} +{"account_number":23,"balance":42374,"firstname":"Kirsten","lastname":"Fox","age":20,"gender":"M","address":"330 Dumont Avenue","employer":"Codax","email":"kirstenfox@codax.com","city":"Walton","state":"AK","_is_real":true} +{"index":{"_id":"28"}} +{"account_number":28,"balance":42112,"firstname":"Vega","lastname":"Flynn","age":20,"gender":"M","address":"647 Hyman Court","employer":"Accupharm","email":"vegaflynn@accupharm.com","city":"Masthope","state":"OH","_is_real":true} +{"index":{"_id":"30"}} +{"account_number":30,"balance":19087,"firstname":"Lamb","lastname":"Townsend","age":26,"gender":"M","address":"169 Lyme Avenue","employer":"Geeknet","email":"lambtownsend@geeknet.com","city":"Epworth","state":"AL","_is_real":true} +{"index":{"_id":"35"}} +{"account_number":35,"balance":42039,"firstname":"Darla","lastname":"Bridges","age":27,"gender":"F","address":"315 Central Avenue","employer":"Xeronk","email":"darlabridges@xeronk.com","city":"Woodlake","state":"RI","_is_real":true} +{"index":{"_id":"42"}} +{"account_number":42,"balance":21137,"firstname":"Harding","lastname":"Hobbs","age":26,"gender":"F","address":"474 Ridgewood Place","employer":"Xth","email":"hardinghobbs@xth.com","city":"Heil","state":"ND","_is_real":true} +{"index":{"_id":"47"}} +{"account_number":47,"balance":33044,"firstname":"Georgia","lastname":"Wilkerson","age":23,"gender":"M","address":"369 Herbert Street","employer":"Endipin","email":"georgiawilkerson@endipin.com","city":"Dellview","state":"WI","_is_real":true} +{"index":{"_id":"54"}} +{"account_number":54,"balance":23406,"firstname":"Angel","lastname":"Mann","age":22,"gender":"F","address":"229 Ferris Street","employer":"Amtas","email":"angelmann@amtas.com","city":"Calverton","state":"WA","_is_real":true} +{"index":{"_id":"59"}} +{"account_number":59,"balance":37728,"firstname":"Malone","lastname":"Justice","age":37,"gender":"F","address":"721 Russell Street","employer":"Emoltra","email":"malonejustice@emoltra.com","city":"Trucksville","state":"HI","_is_real":true} +{"index":{"_id":"61"}} +{"account_number":61,"balance":6856,"firstname":"Shawn","lastname":"Baird","age":20,"gender":"M","address":"605 Monument Walk","employer":"Moltonic","email":"shawnbaird@moltonic.com","city":"Darlington","state":"MN","_is_real":true} +{"index":{"_id":"66"}} +{"account_number":66,"balance":25939,"firstname":"Franks","lastname":"Salinas","age":28,"gender":"M","address":"437 Hamilton Walk","employer":"Cowtown","email":"frankssalinas@cowtown.com","city":"Chase","state":"VT","_is_real":true} +{"index":{"_id":"73"}} +{"account_number":73,"balance":33457,"firstname":"Irene","lastname":"Stephenson","age":32,"gender":"M","address":"684 Miller Avenue","employer":"Hawkster","email":"irenestephenson@hawkster.com","city":"Levant","state":"AR","_is_real":true} +{"index":{"_id":"78"}} +{"account_number":78,"balance":48656,"firstname":"Elvira","lastname":"Patterson","age":23,"gender":"F","address":"834 Amber Street","employer":"Assistix","email":"elvirapatterson@assistix.com","city":"Dunbar","state":"TN","_is_real":true} +{"index":{"_id":"80"}} +{"account_number":80,"balance":13445,"firstname":"Lacey","lastname":"Blanchard","age":30,"gender":"F","address":"823 Himrod Street","employer":"Comdom","email":"laceyblanchard@comdom.com","city":"Matthews","state":"MO","_is_real":true} +{"index":{"_id":"85"}} +{"account_number":85,"balance":48735,"firstname":"Wilcox","lastname":"Sellers","age":20,"gender":"M","address":"212 Irving Avenue","employer":"Confrenzy","email":"wilcoxsellers@confrenzy.com","city":"Kipp","state":"MT","_is_real":true} +{"index":{"_id":"92"}} +{"account_number":92,"balance":26753,"firstname":"Gay","lastname":"Brewer","age":34,"gender":"M","address":"369 Ditmars Street","employer":"Savvy","email":"gaybrewer@savvy.com","city":"Moquino","state":"HI","_is_real":true} +{"index":{"_id":"97"}} +{"account_number":97,"balance":49671,"firstname":"Karen","lastname":"Trujillo","age":40,"gender":"F","address":"512 Cumberland Walk","employer":"Tsunamia","email":"karentrujillo@tsunamia.com","city":"Fredericktown","state":"MO","_is_real":true} +{"index":{"_id":"100"}} +{"account_number":100,"balance":29869,"firstname":"Madden","lastname":"Woods","age":32,"gender":"F","address":"696 Ryder Avenue","employer":"Slumberia","email":"maddenwoods@slumberia.com","city":"Deercroft","state":"ME","_is_real":true} +{"index":{"_id":"105"}} +{"account_number":105,"balance":29654,"firstname":"Castillo","lastname":"Dickerson","age":33,"gender":"F","address":"673 Oxford Street","employer":"Tellifly","email":"castillodickerson@tellifly.com","city":"Succasunna","state":"NY","_is_real":true} +{"index":{"_id":"112"}} +{"account_number":112,"balance":38395,"firstname":"Frederick","lastname":"Case","age":30,"gender":"F","address":"580 Lexington Avenue","employer":"Talkalot","email":"frederickcase@talkalot.com","city":"Orovada","state":"MA","_is_real":true} +{"index":{"_id":"117"}} +{"account_number":117,"balance":48831,"firstname":"Robin","lastname":"Hays","age":38,"gender":"F","address":"347 Hornell Loop","employer":"Pasturia","email":"robinhays@pasturia.com","city":"Sims","state":"WY","_is_real":true} +{"index":{"_id":"124"}} +{"account_number":124,"balance":16425,"firstname":"Fern","lastname":"Lambert","age":20,"gender":"M","address":"511 Jay Street","employer":"Furnitech","email":"fernlambert@furnitech.com","city":"Cloverdale","state":"FL","_is_real":true} +{"index":{"_id":"129"}} +{"account_number":129,"balance":42409,"firstname":"Alexandria","lastname":"Sanford","age":33,"gender":"F","address":"934 Ridgecrest Terrace","employer":"Kyagoro","email":"alexandriasanford@kyagoro.com","city":"Concho","state":"UT","_is_real":true} +{"index":{"_id":"131"}} +{"account_number":131,"balance":28030,"firstname":"Dollie","lastname":"Koch","age":22,"gender":"F","address":"287 Manhattan Avenue","employer":"Skinserve","email":"dolliekoch@skinserve.com","city":"Shasta","state":"PA","_is_real":true} +{"index":{"_id":"136"}} +{"account_number":136,"balance":45801,"firstname":"Winnie","lastname":"Holland","age":38,"gender":"M","address":"198 Mill Lane","employer":"Neteria","email":"winnieholland@neteria.com","city":"Urie","state":"IL","_is_real":true} +{"index":{"_id":"143"}} +{"account_number":143,"balance":43093,"firstname":"Cohen","lastname":"Noble","age":39,"gender":"M","address":"454 Nelson Street","employer":"Buzzworks","email":"cohennoble@buzzworks.com","city":"Norvelt","state":"CO","_is_real":true} +{"index":{"_id":"148"}} +{"account_number":148,"balance":3662,"firstname":"Annmarie","lastname":"Snider","age":34,"gender":"F","address":"857 Lafayette Walk","employer":"Edecine","email":"annmariesnider@edecine.com","city":"Hollins","state":"OH","_is_real":true} +{"index":{"_id":"150"}} +{"account_number":150,"balance":15306,"firstname":"Ortega","lastname":"Dalton","age":20,"gender":"M","address":"237 Mermaid Avenue","employer":"Rameon","email":"ortegadalton@rameon.com","city":"Maxville","state":"NH","_is_real":true} +{"index":{"_id":"155"}} +{"account_number":155,"balance":27878,"firstname":"Atkinson","lastname":"Hudson","age":39,"gender":"F","address":"434 Colin Place","employer":"Qualitern","email":"atkinsonhudson@qualitern.com","city":"Hoehne","state":"OH","_is_real":true} +{"index":{"_id":"162"}} +{"account_number":162,"balance":6302,"firstname":"Griffith","lastname":"Calderon","age":35,"gender":"M","address":"871 Vandervoort Place","employer":"Quotezart","email":"griffithcalderon@quotezart.com","city":"Barclay","state":"FL","_is_real":true} +{"index":{"_id":"167"}} +{"account_number":167,"balance":42051,"firstname":"Hampton","lastname":"Ryan","age":20,"gender":"M","address":"618 Fleet Place","employer":"Zipak","email":"hamptonryan@zipak.com","city":"Irwin","state":"KS","_is_real":true} +{"index":{"_id":"174"}} +{"account_number":174,"balance":1464,"firstname":"Gamble","lastname":"Pierce","age":23,"gender":"F","address":"650 Eagle Street","employer":"Matrixity","email":"gamblepierce@matrixity.com","city":"Abiquiu","state":"OR","_is_real":true} +{"index":{"_id":"179"}} +{"account_number":179,"balance":13265,"firstname":"Elise","lastname":"Drake","age":25,"gender":"M","address":"305 Christopher Avenue","employer":"Turnling","email":"elisedrake@turnling.com","city":"Loretto","state":"LA","_is_real":true} +{"index":{"_id":"181"}} +{"account_number":181,"balance":27983,"firstname":"Bennett","lastname":"Hampton","age":22,"gender":"F","address":"435 Billings Place","employer":"Voipa","email":"bennetthampton@voipa.com","city":"Rodman","state":"WY","_is_real":true} +{"index":{"_id":"186"}} +{"account_number":186,"balance":18373,"firstname":"Kline","lastname":"Joyce","age":32,"gender":"M","address":"285 Falmouth Street","employer":"Tetratrex","email":"klinejoyce@tetratrex.com","city":"Klondike","state":"SD","_is_real":true} +{"index":{"_id":"193"}} +{"account_number":193,"balance":13412,"firstname":"Patty","lastname":"Petty","age":34,"gender":"F","address":"251 Vermont Street","employer":"Kinetica","email":"pattypetty@kinetica.com","city":"Grantville","state":"MS","_is_real":true} +{"index":{"_id":"198"}} +{"account_number":198,"balance":19686,"firstname":"Rachael","lastname":"Sharp","age":38,"gender":"F","address":"443 Vernon Avenue","employer":"Powernet","email":"rachaelsharp@powernet.com","city":"Canoochee","state":"UT","_is_real":true} +{"index":{"_id":"201"}} +{"account_number":201,"balance":14586,"firstname":"Ronda","lastname":"Perry","age":25,"gender":"F","address":"856 Downing Street","employer":"Artiq","email":"rondaperry@artiq.com","city":"Colton","state":"WV","_is_real":true} +{"index":{"_id":"206"}} +{"account_number":206,"balance":47423,"firstname":"Kelli","lastname":"Francis","age":20,"gender":"M","address":"671 George Street","employer":"Exoswitch","email":"kellifrancis@exoswitch.com","city":"Babb","state":"NJ","_is_real":true} +{"index":{"_id":"213"}} +{"account_number":213,"balance":34172,"firstname":"Bauer","lastname":"Summers","age":27,"gender":"M","address":"257 Boynton Place","employer":"Voratak","email":"bauersummers@voratak.com","city":"Oceola","state":"NC","_is_real":true} +{"index":{"_id":"218"}} +{"account_number":218,"balance":26702,"firstname":"Garrison","lastname":"Bryan","age":24,"gender":"F","address":"478 Greenpoint Avenue","employer":"Uniworld","email":"garrisonbryan@uniworld.com","city":"Comptche","state":"WI","_is_real":true} +{"index":{"_id":"220"}} +{"account_number":220,"balance":3086,"firstname":"Tania","lastname":"Middleton","age":22,"gender":"F","address":"541 Gunther Place","employer":"Zerology","email":"taniamiddleton@zerology.com","city":"Linwood","state":"IN","_is_real":true} +{"index":{"_id":"225"}} +{"account_number":225,"balance":21949,"firstname":"Maryann","lastname":"Murphy","age":24,"gender":"F","address":"894 Bridgewater Street","employer":"Cinesanct","email":"maryannmurphy@cinesanct.com","city":"Cartwright","state":"RI","_is_real":true} +{"index":{"_id":"232"}} +{"account_number":232,"balance":11984,"firstname":"Carr","lastname":"Jensen","age":34,"gender":"F","address":"995 Micieli Place","employer":"Biohab","email":"carrjensen@biohab.com","city":"Waikele","state":"OH","_is_real":true} +{"index":{"_id":"237"}} +{"account_number":237,"balance":5603,"firstname":"Kirby","lastname":"Watkins","age":27,"gender":"F","address":"348 Blake Court","employer":"Sonique","email":"kirbywatkins@sonique.com","city":"Freelandville","state":"PA","_is_real":true} +{"index":{"_id":"244"}} +{"account_number":244,"balance":8048,"firstname":"Judith","lastname":"Riggs","age":27,"gender":"F","address":"590 Kosciusko Street","employer":"Arctiq","email":"judithriggs@arctiq.com","city":"Gorham","state":"DC","_is_real":true} +{"index":{"_id":"249"}} +{"account_number":249,"balance":16822,"firstname":"Mckinney","lastname":"Gallagher","age":38,"gender":"F","address":"939 Seigel Court","employer":"Premiant","email":"mckinneygallagher@premiant.com","city":"Catharine","state":"NH","_is_real":true} +{"index":{"_id":"251"}} +{"account_number":251,"balance":13475,"firstname":"Marks","lastname":"Graves","age":39,"gender":"F","address":"427 Lawn Court","employer":"Dentrex","email":"marksgraves@dentrex.com","city":"Waukeenah","state":"IL","_is_real":true} +{"index":{"_id":"256"}} +{"account_number":256,"balance":48318,"firstname":"Simon","lastname":"Hogan","age":31,"gender":"M","address":"789 Suydam Place","employer":"Dancerity","email":"simonhogan@dancerity.com","city":"Dargan","state":"GA","_is_real":true} +{"index":{"_id":"263"}} +{"account_number":263,"balance":12837,"firstname":"Thornton","lastname":"Meyer","age":29,"gender":"M","address":"575 Elliott Place","employer":"Peticular","email":"thorntonmeyer@peticular.com","city":"Dotsero","state":"NH","_is_real":true} +{"index":{"_id":"268"}} +{"account_number":268,"balance":20925,"firstname":"Avis","lastname":"Blackwell","age":36,"gender":"M","address":"569 Jerome Avenue","employer":"Magnina","email":"avisblackwell@magnina.com","city":"Bethany","state":"MD","_is_real":true} +{"index":{"_id":"270"}} +{"account_number":270,"balance":43951,"firstname":"Moody","lastname":"Harmon","age":39,"gender":"F","address":"233 Vanderbilt Street","employer":"Otherside","email":"moodyharmon@otherside.com","city":"Elwood","state":"MT","_is_real":true} +{"index":{"_id":"275"}} +{"account_number":275,"balance":2384,"firstname":"Reynolds","lastname":"Barnett","age":31,"gender":"M","address":"394 Stockton Street","employer":"Austex","email":"reynoldsbarnett@austex.com","city":"Grandview","state":"MS","_is_real":true} +{"index":{"_id":"282"}} +{"account_number":282,"balance":38540,"firstname":"Gay","lastname":"Schultz","age":25,"gender":"F","address":"805 Claver Place","employer":"Handshake","email":"gayschultz@handshake.com","city":"Tampico","state":"MA","_is_real":true} +{"index":{"_id":"287"}} +{"account_number":287,"balance":10845,"firstname":"Valerie","lastname":"Lang","age":35,"gender":"F","address":"423 Midwood Street","employer":"Quarx","email":"valerielang@quarx.com","city":"Cannondale","state":"VT","_is_real":true} +{"index":{"_id":"294"}} +{"account_number":294,"balance":29582,"firstname":"Pitts","lastname":"Haynes","age":26,"gender":"M","address":"901 Broome Street","employer":"Aquazure","email":"pittshaynes@aquazure.com","city":"Turah","state":"SD","_is_real":true} +{"index":{"_id":"299"}} +{"account_number":299,"balance":40825,"firstname":"Angela","lastname":"Talley","age":36,"gender":"F","address":"822 Bills Place","employer":"Remold","email":"angelatalley@remold.com","city":"Bethpage","state":"DC","_is_real":true} +{"index":{"_id":"302"}} +{"account_number":302,"balance":11298,"firstname":"Isabella","lastname":"Hewitt","age":40,"gender":"M","address":"455 Bedford Avenue","employer":"Cincyr","email":"isabellahewitt@cincyr.com","city":"Blanford","state":"IN","_is_real":true} +{"index":{"_id":"307"}} +{"account_number":307,"balance":43355,"firstname":"Enid","lastname":"Ashley","age":23,"gender":"M","address":"412 Emerson Place","employer":"Avenetro","email":"enidashley@avenetro.com","city":"Catherine","state":"WI","_is_real":true} +{"index":{"_id":"314"}} +{"account_number":314,"balance":5848,"firstname":"Norton","lastname":"Norton","age":35,"gender":"M","address":"252 Ditmas Avenue","employer":"Talkola","email":"nortonnorton@talkola.com","city":"Veyo","state":"SC","_is_real":true} +{"index":{"_id":"319"}} +{"account_number":319,"balance":15430,"firstname":"Ferrell","lastname":"Mckinney","age":36,"gender":"M","address":"874 Cranberry Street","employer":"Portaline","email":"ferrellmckinney@portaline.com","city":"Rose","state":"WV","_is_real":true} +{"index":{"_id":"321"}} +{"account_number":321,"balance":43370,"firstname":"Marta","lastname":"Larsen","age":35,"gender":"M","address":"617 Williams Court","employer":"Manufact","email":"martalarsen@manufact.com","city":"Sisquoc","state":"MA","_is_real":true} +{"index":{"_id":"326"}} +{"account_number":326,"balance":9692,"firstname":"Pearl","lastname":"Reese","age":30,"gender":"F","address":"451 Colonial Court","employer":"Accruex","email":"pearlreese@accruex.com","city":"Westmoreland","state":"MD","_is_real":true} +{"index":{"_id":"333"}} +{"account_number":333,"balance":22778,"firstname":"Trudy","lastname":"Sweet","age":27,"gender":"F","address":"881 Kiely Place","employer":"Acumentor","email":"trudysweet@acumentor.com","city":"Kent","state":"IA","_is_real":true} +{"index":{"_id":"338"}} +{"account_number":338,"balance":6969,"firstname":"Pierce","lastname":"Lawrence","age":35,"gender":"M","address":"318 Gallatin Place","employer":"Lunchpad","email":"piercelawrence@lunchpad.com","city":"Iola","state":"MD","_is_real":true} +{"index":{"_id":"340"}} +{"account_number":340,"balance":42072,"firstname":"Juarez","lastname":"Gutierrez","age":40,"gender":"F","address":"802 Seba Avenue","employer":"Billmed","email":"juarezgutierrez@billmed.com","city":"Malott","state":"OH","_is_real":true} +{"index":{"_id":"345"}} +{"account_number":345,"balance":9812,"firstname":"Parker","lastname":"Hines","age":38,"gender":"M","address":"715 Mill Avenue","employer":"Baluba","email":"parkerhines@baluba.com","city":"Blackgum","state":"KY","_is_real":true} +{"index":{"_id":"352"}} +{"account_number":352,"balance":20290,"firstname":"Kendra","lastname":"Mcintosh","age":31,"gender":"F","address":"963 Wolf Place","employer":"Orboid","email":"kendramcintosh@orboid.com","city":"Bladensburg","state":"AK","_is_real":true} +{"index":{"_id":"357"}} +{"account_number":357,"balance":15102,"firstname":"Adele","lastname":"Carroll","age":39,"gender":"F","address":"381 Arion Place","employer":"Aquafire","email":"adelecarroll@aquafire.com","city":"Springville","state":"RI","_is_real":true} +{"index":{"_id":"364"}} +{"account_number":364,"balance":35247,"firstname":"Felicia","lastname":"Merrill","age":40,"gender":"F","address":"229 Branton Street","employer":"Prosely","email":"feliciamerrill@prosely.com","city":"Dola","state":"MA","_is_real":true} +{"index":{"_id":"369"}} +{"account_number":369,"balance":17047,"firstname":"Mcfadden","lastname":"Guy","age":28,"gender":"F","address":"445 Lott Avenue","employer":"Kangle","email":"mcfaddenguy@kangle.com","city":"Greenbackville","state":"DE","_is_real":true} +{"index":{"_id":"371"}} +{"account_number":371,"balance":19751,"firstname":"Barker","lastname":"Allen","age":32,"gender":"F","address":"295 Wallabout Street","employer":"Nexgene","email":"barkerallen@nexgene.com","city":"Nanafalia","state":"NE","_is_real":true} +{"index":{"_id":"376"}} +{"account_number":376,"balance":44407,"firstname":"Mcmillan","lastname":"Dunn","age":21,"gender":"F","address":"771 Dorchester Road","employer":"Eargo","email":"mcmillandunn@eargo.com","city":"Yogaville","state":"RI","_is_real":true} +{"index":{"_id":"383"}} +{"account_number":383,"balance":48889,"firstname":"Knox","lastname":"Larson","age":28,"gender":"F","address":"962 Bartlett Place","employer":"Bostonic","email":"knoxlarson@bostonic.com","city":"Smeltertown","state":"TX","_is_real":true} +{"index":{"_id":"388"}} +{"account_number":388,"balance":9606,"firstname":"Julianne","lastname":"Nicholson","age":26,"gender":"F","address":"338 Crescent Street","employer":"Viasia","email":"juliannenicholson@viasia.com","city":"Alleghenyville","state":"MO","_is_real":true} +{"index":{"_id":"390"}} +{"account_number":390,"balance":7464,"firstname":"Ramona","lastname":"Roy","age":32,"gender":"M","address":"135 Banner Avenue","employer":"Deminimum","email":"ramonaroy@deminimum.com","city":"Dodge","state":"ID","_is_real":true} +{"index":{"_id":"395"}} +{"account_number":395,"balance":18679,"firstname":"Juliet","lastname":"Whitaker","age":31,"gender":"M","address":"128 Remsen Avenue","employer":"Toyletry","email":"julietwhitaker@toyletry.com","city":"Yonah","state":"LA","_is_real":true} +{"index":{"_id":"403"}} +{"account_number":403,"balance":18833,"firstname":"Williamson","lastname":"Horn","age":32,"gender":"M","address":"223 Strickland Avenue","employer":"Nimon","email":"williamsonhorn@nimon.com","city":"Bawcomville","state":"NJ","_is_real":true} +{"index":{"_id":"408"}} +{"account_number":408,"balance":34666,"firstname":"Lidia","lastname":"Guerrero","age":30,"gender":"M","address":"254 Stratford Road","employer":"Snowpoke","email":"lidiaguerrero@snowpoke.com","city":"Fairlee","state":"LA","_is_real":true} +{"index":{"_id":"410"}} +{"account_number":410,"balance":31200,"firstname":"Fox","lastname":"Cardenas","age":39,"gender":"M","address":"987 Monitor Street","employer":"Corpulse","email":"foxcardenas@corpulse.com","city":"Southview","state":"NE","_is_real":true} +{"index":{"_id":"415"}} +{"account_number":415,"balance":19449,"firstname":"Martinez","lastname":"Benson","age":36,"gender":"M","address":"172 Berkeley Place","employer":"Enersol","email":"martinezbenson@enersol.com","city":"Chumuckla","state":"AL","_is_real":true} +{"index":{"_id":"422"}} +{"account_number":422,"balance":40162,"firstname":"Brigitte","lastname":"Scott","age":26,"gender":"M","address":"662 Vermont Court","employer":"Waretel","email":"brigittescott@waretel.com","city":"Elrama","state":"VA","_is_real":true} +{"index":{"_id":"427"}} +{"account_number":427,"balance":1463,"firstname":"Rebekah","lastname":"Garrison","age":36,"gender":"F","address":"837 Hampton Avenue","employer":"Niquent","email":"rebekahgarrison@niquent.com","city":"Zarephath","state":"NY","_is_real":true} +{"index":{"_id":"434"}} +{"account_number":434,"balance":11329,"firstname":"Christa","lastname":"Huff","age":25,"gender":"M","address":"454 Oriental Boulevard","employer":"Earthpure","email":"christahuff@earthpure.com","city":"Stevens","state":"DC","_is_real":true} +{"index":{"_id":"439"}} +{"account_number":439,"balance":22752,"firstname":"Lula","lastname":"Williams","age":35,"gender":"M","address":"630 Furman Avenue","employer":"Vinch","email":"lulawilliams@vinch.com","city":"Newcastle","state":"ME","_is_real":true} +{"index":{"_id":"441"}} +{"account_number":441,"balance":47947,"firstname":"Dickson","lastname":"Mcgee","age":29,"gender":"M","address":"478 Knight Court","employer":"Gogol","email":"dicksonmcgee@gogol.com","city":"Laurelton","state":"AR","_is_real":true} +{"index":{"_id":"446"}} +{"account_number":446,"balance":23071,"firstname":"Lolita","lastname":"Fleming","age":32,"gender":"F","address":"918 Bridge Street","employer":"Vidto","email":"lolitafleming@vidto.com","city":"Brownlee","state":"HI","_is_real":true} +{"index":{"_id":"453"}} +{"account_number":453,"balance":21520,"firstname":"Hood","lastname":"Powell","age":24,"gender":"F","address":"479 Brevoort Place","employer":"Vortexaco","email":"hoodpowell@vortexaco.com","city":"Alderpoint","state":"CT","_is_real":true} +{"index":{"_id":"458"}} +{"account_number":458,"balance":8865,"firstname":"Aida","lastname":"Wolf","age":21,"gender":"F","address":"403 Thames Street","employer":"Isis","email":"aidawolf@isis.com","city":"Bordelonville","state":"ME","_is_real":true} +{"index":{"_id":"460"}} +{"account_number":460,"balance":37734,"firstname":"Aguirre","lastname":"White","age":21,"gender":"F","address":"190 Crooke Avenue","employer":"Unq","email":"aguirrewhite@unq.com","city":"Albany","state":"NJ","_is_real":true} +{"index":{"_id":"465"}} +{"account_number":465,"balance":10681,"firstname":"Pearlie","lastname":"Holman","age":29,"gender":"M","address":"916 Evergreen Avenue","employer":"Hometown","email":"pearlieholman@hometown.com","city":"Needmore","state":"UT","_is_real":true} +{"index":{"_id":"472"}} +{"account_number":472,"balance":25571,"firstname":"Lee","lastname":"Long","age":32,"gender":"F","address":"288 Mill Street","employer":"Comverges","email":"leelong@comverges.com","city":"Movico","state":"MT","_is_real":true} +{"index":{"_id":"477"}} +{"account_number":477,"balance":25892,"firstname":"Holcomb","lastname":"Cobb","age":40,"gender":"M","address":"369 Marconi Place","employer":"Steeltab","email":"holcombcobb@steeltab.com","city":"Byrnedale","state":"CA","_is_real":true} +{"index":{"_id":"484"}} +{"account_number":484,"balance":3274,"firstname":"Staci","lastname":"Melendez","age":35,"gender":"F","address":"751 Otsego Street","employer":"Namebox","email":"stacimelendez@namebox.com","city":"Harborton","state":"NV","_is_real":true} +{"index":{"_id":"489"}} +{"account_number":489,"balance":7879,"firstname":"Garrett","lastname":"Langley","age":36,"gender":"M","address":"331 Bowne Street","employer":"Zillidium","email":"garrettlangley@zillidium.com","city":"Riviera","state":"LA","_is_real":true} +{"index":{"_id":"491"}} +{"account_number":491,"balance":42942,"firstname":"Teresa","lastname":"Owen","age":24,"gender":"F","address":"713 Canton Court","employer":"Plasmos","email":"teresaowen@plasmos.com","city":"Bartonsville","state":"NH","_is_real":true} +{"index":{"_id":"496"}} +{"account_number":496,"balance":14869,"firstname":"Alison","lastname":"Conrad","age":35,"gender":"F","address":"347 Varet Street","employer":"Perkle","email":"alisonconrad@perkle.com","city":"Cliffside","state":"OH","_is_real":true} +{"index":{"_id":"504"}} +{"account_number":504,"balance":49205,"firstname":"Shanna","lastname":"Chambers","age":23,"gender":"M","address":"220 Beard Street","employer":"Corporana","email":"shannachambers@corporana.com","city":"Cashtown","state":"AZ","_is_real":true} +{"index":{"_id":"509"}} +{"account_number":509,"balance":34754,"firstname":"Durham","lastname":"Pacheco","age":40,"gender":"M","address":"129 Plymouth Street","employer":"Datacator","email":"durhampacheco@datacator.com","city":"Loveland","state":"NC","_is_real":true} +{"index":{"_id":"511"}} +{"account_number":511,"balance":40908,"firstname":"Elba","lastname":"Grant","age":24,"gender":"F","address":"157 Bijou Avenue","employer":"Dognost","email":"elbagrant@dognost.com","city":"Coyote","state":"MT","_is_real":true} +{"index":{"_id":"516"}} +{"account_number":516,"balance":44940,"firstname":"Roy","lastname":"Smith","age":37,"gender":"M","address":"770 Cherry Street","employer":"Parleynet","email":"roysmith@parleynet.com","city":"Carrsville","state":"RI","_is_real":true} +{"index":{"_id":"523"}} +{"account_number":523,"balance":28729,"firstname":"Amalia","lastname":"Benjamin","age":40,"gender":"F","address":"173 Bushwick Place","employer":"Sentia","email":"amaliabenjamin@sentia.com","city":"Jacumba","state":"OK","_is_real":true} +{"index":{"_id":"528"}} +{"account_number":528,"balance":4071,"firstname":"Thompson","lastname":"Hoover","age":27,"gender":"F","address":"580 Garden Street","employer":"Portalis","email":"thompsonhoover@portalis.com","city":"Knowlton","state":"AL","_is_real":true} +{"index":{"_id":"530"}} +{"account_number":530,"balance":8840,"firstname":"Kathrine","lastname":"Evans","age":37,"gender":"M","address":"422 Division Place","employer":"Spherix","email":"kathrineevans@spherix.com","city":"Biddle","state":"CO","_is_real":true} +{"index":{"_id":"535"}} +{"account_number":535,"balance":8715,"firstname":"Fry","lastname":"George","age":34,"gender":"M","address":"722 Green Street","employer":"Ewaves","email":"frygeorge@ewaves.com","city":"Kenmar","state":"DE","_is_real":true} +{"index":{"_id":"542"}} +{"account_number":542,"balance":23285,"firstname":"Michelle","lastname":"Mayo","age":35,"gender":"M","address":"657 Caton Place","employer":"Biflex","email":"michellemayo@biflex.com","city":"Beaverdale","state":"WY","_is_real":true} +{"index":{"_id":"547"}} +{"account_number":547,"balance":12870,"firstname":"Eaton","lastname":"Rios","age":32,"gender":"M","address":"744 Withers Street","employer":"Podunk","email":"eatonrios@podunk.com","city":"Chelsea","state":"IA","_is_real":true} +{"index":{"_id":"554"}} +{"account_number":554,"balance":33163,"firstname":"Townsend","lastname":"Atkins","age":39,"gender":"M","address":"566 Ira Court","employer":"Acruex","email":"townsendatkins@acruex.com","city":"Valle","state":"IA","_is_real":true} +{"index":{"_id":"559"}} +{"account_number":559,"balance":11450,"firstname":"Tonia","lastname":"Schmidt","age":38,"gender":"F","address":"508 Sheffield Avenue","employer":"Extro","email":"toniaschmidt@extro.com","city":"Newry","state":"CT","_is_real":true} +{"index":{"_id":"561"}} +{"account_number":561,"balance":12370,"firstname":"Sellers","lastname":"Davis","age":30,"gender":"M","address":"860 Madoc Avenue","employer":"Isodrive","email":"sellersdavis@isodrive.com","city":"Trail","state":"KS","_is_real":true} +{"index":{"_id":"566"}} +{"account_number":566,"balance":6183,"firstname":"Cox","lastname":"Roman","age":37,"gender":"M","address":"349 Winthrop Street","employer":"Medcom","email":"coxroman@medcom.com","city":"Rosewood","state":"WY","_is_real":true} +{"index":{"_id":"573"}} +{"account_number":573,"balance":32171,"firstname":"Callie","lastname":"Castaneda","age":36,"gender":"M","address":"799 Scott Avenue","employer":"Earthwax","email":"calliecastaneda@earthwax.com","city":"Marshall","state":"NH","_is_real":true} +{"index":{"_id":"578"}} +{"account_number":578,"balance":34259,"firstname":"Holmes","lastname":"Mcknight","age":37,"gender":"M","address":"969 Metropolitan Avenue","employer":"Cubicide","email":"holmesmcknight@cubicide.com","city":"Aguila","state":"PA","_is_real":true} +{"index":{"_id":"580"}} +{"account_number":580,"balance":13716,"firstname":"Mcmahon","lastname":"York","age":34,"gender":"M","address":"475 Beacon Court","employer":"Zillar","email":"mcmahonyork@zillar.com","city":"Farmington","state":"MO","_is_real":true} +{"index":{"_id":"585"}} +{"account_number":585,"balance":26745,"firstname":"Nieves","lastname":"Nolan","age":32,"gender":"M","address":"115 Seagate Terrace","employer":"Jumpstack","email":"nievesnolan@jumpstack.com","city":"Eastmont","state":"UT","_is_real":true} +{"index":{"_id":"592"}} +{"account_number":592,"balance":32968,"firstname":"Head","lastname":"Webster","age":36,"gender":"F","address":"987 Lefferts Avenue","employer":"Empirica","email":"headwebster@empirica.com","city":"Rockingham","state":"TN","_is_real":true} +{"index":{"_id":"597"}} +{"account_number":597,"balance":11246,"firstname":"Penny","lastname":"Knowles","age":33,"gender":"M","address":"139 Forbell Street","employer":"Ersum","email":"pennyknowles@ersum.com","city":"Vallonia","state":"IA","_is_real":true} +{"index":{"_id":"600"}} +{"account_number":600,"balance":10336,"firstname":"Simmons","lastname":"Byers","age":37,"gender":"M","address":"250 Dictum Court","employer":"Qualitex","email":"simmonsbyers@qualitex.com","city":"Wanship","state":"OH","_is_real":true} +{"index":{"_id":"605"}} +{"account_number":605,"balance":38427,"firstname":"Mcclain","lastname":"Manning","age":24,"gender":"M","address":"832 Leonard Street","employer":"Qiao","email":"mcclainmanning@qiao.com","city":"Calvary","state":"TX","_is_real":true} +{"index":{"_id":"612"}} +{"account_number":612,"balance":11868,"firstname":"Dunn","lastname":"Cameron","age":32,"gender":"F","address":"156 Lorimer Street","employer":"Isonus","email":"dunncameron@isonus.com","city":"Virgie","state":"ND","_is_real":true} +{"index":{"_id":"617"}} +{"account_number":617,"balance":35445,"firstname":"Kitty","lastname":"Cooley","age":22,"gender":"M","address":"788 Seagate Avenue","employer":"Ultrimax","email":"kittycooley@ultrimax.com","city":"Clarktown","state":"MD","_is_real":true} +{"index":{"_id":"624"}} +{"account_number":624,"balance":27538,"firstname":"Roxanne","lastname":"Franklin","age":39,"gender":"F","address":"299 Woodrow Court","employer":"Silodyne","email":"roxannefranklin@silodyne.com","city":"Roulette","state":"VA","_is_real":true} +{"index":{"_id":"629"}} +{"account_number":629,"balance":32987,"firstname":"Mcclure","lastname":"Rodgers","age":26,"gender":"M","address":"806 Pierrepont Place","employer":"Elita","email":"mcclurerodgers@elita.com","city":"Brownsville","state":"MI","_is_real":true} +{"index":{"_id":"631"}} +{"account_number":631,"balance":21657,"firstname":"Corrine","lastname":"Barber","age":32,"gender":"F","address":"447 Hunts Lane","employer":"Quarmony","email":"corrinebarber@quarmony.com","city":"Wyano","state":"IL","_is_real":true} +{"index":{"_id":"636"}} +{"account_number":636,"balance":8036,"firstname":"Agnes","lastname":"Hooper","age":25,"gender":"M","address":"865 Hanson Place","employer":"Digial","email":"agneshooper@digial.com","city":"Sperryville","state":"OK","_is_real":true} +{"index":{"_id":"643"}} +{"account_number":643,"balance":8057,"firstname":"Hendricks","lastname":"Stokes","age":23,"gender":"F","address":"142 Barbey Street","employer":"Remotion","email":"hendricksstokes@remotion.com","city":"Lewis","state":"MA","_is_real":true} +{"index":{"_id":"648"}} +{"account_number":648,"balance":11506,"firstname":"Terry","lastname":"Montgomery","age":21,"gender":"F","address":"115 Franklin Avenue","employer":"Enervate","email":"terrymontgomery@enervate.com","city":"Bascom","state":"MA","_is_real":true} +{"index":{"_id":"650"}} +{"account_number":650,"balance":18091,"firstname":"Benton","lastname":"Knight","age":28,"gender":"F","address":"850 Aitken Place","employer":"Pholio","email":"bentonknight@pholio.com","city":"Cobbtown","state":"AL","_is_real":true} +{"index":{"_id":"655"}} +{"account_number":655,"balance":22912,"firstname":"Eula","lastname":"Taylor","age":30,"gender":"M","address":"520 Orient Avenue","employer":"Miracula","email":"eulataylor@miracula.com","city":"Wacissa","state":"IN","_is_real":true} +{"index":{"_id":"662"}} +{"account_number":662,"balance":10138,"firstname":"Daisy","lastname":"Burnett","age":33,"gender":"M","address":"114 Norman Avenue","employer":"Liquicom","email":"daisyburnett@liquicom.com","city":"Grahamtown","state":"MD","_is_real":true} +{"index":{"_id":"667"}} +{"account_number":667,"balance":22559,"firstname":"Juliana","lastname":"Chase","age":32,"gender":"M","address":"496 Coleridge Street","employer":"Comtract","email":"julianachase@comtract.com","city":"Wilsonia","state":"NJ","_is_real":true} +{"index":{"_id":"674"}} +{"account_number":674,"balance":36038,"firstname":"Watts","lastname":"Shannon","age":22,"gender":"F","address":"600 Story Street","employer":"Joviold","email":"wattsshannon@joviold.com","city":"Fairhaven","state":"ID","_is_real":true} +{"index":{"_id":"679"}} +{"account_number":679,"balance":20149,"firstname":"Henrietta","lastname":"Bonner","age":33,"gender":"M","address":"461 Bond Street","employer":"Geekol","email":"henriettabonner@geekol.com","city":"Richville","state":"WA","_is_real":true} +{"index":{"_id":"681"}} +{"account_number":681,"balance":34244,"firstname":"Velazquez","lastname":"Wolfe","age":33,"gender":"M","address":"773 Eckford Street","employer":"Zisis","email":"velazquezwolfe@zisis.com","city":"Smock","state":"ME","_is_real":true} +{"index":{"_id":"686"}} +{"account_number":686,"balance":10116,"firstname":"Decker","lastname":"Mcclure","age":30,"gender":"F","address":"236 Commerce Street","employer":"Everest","email":"deckermcclure@everest.com","city":"Gibbsville","state":"TN","_is_real":true} +{"index":{"_id":"693"}} +{"account_number":693,"balance":31233,"firstname":"Tabatha","lastname":"Zimmerman","age":30,"gender":"F","address":"284 Emmons Avenue","employer":"Pushcart","email":"tabathazimmerman@pushcart.com","city":"Esmont","state":"NC","_is_real":true} +{"index":{"_id":"698"}} +{"account_number":698,"balance":14965,"firstname":"Baker","lastname":"Armstrong","age":36,"gender":"F","address":"796 Tehama Street","employer":"Nurplex","email":"bakerarmstrong@nurplex.com","city":"Starks","state":"UT","_is_real":true} +{"index":{"_id":"701"}} +{"account_number":701,"balance":23772,"firstname":"Gardner","lastname":"Griffith","age":27,"gender":"M","address":"187 Moore Place","employer":"Vertide","email":"gardnergriffith@vertide.com","city":"Coventry","state":"NV","_is_real":true} +{"index":{"_id":"706"}} +{"account_number":706,"balance":5282,"firstname":"Eliza","lastname":"Potter","age":39,"gender":"M","address":"945 Dunham Place","employer":"Playce","email":"elizapotter@playce.com","city":"Woodruff","state":"AK","_is_real":true} +{"index":{"_id":"713"}} +{"account_number":713,"balance":20054,"firstname":"Iris","lastname":"Mcguire","age":21,"gender":"F","address":"508 Benson Avenue","employer":"Duflex","email":"irismcguire@duflex.com","city":"Hillsboro","state":"MO","_is_real":true} +{"index":{"_id":"718"}} +{"account_number":718,"balance":13876,"firstname":"Hickman","lastname":"Dillard","age":22,"gender":"F","address":"132 Etna Street","employer":"Genmy","email":"hickmandillard@genmy.com","city":"Curtice","state":"NV","_is_real":true} +{"index":{"_id":"720"}} +{"account_number":720,"balance":31356,"firstname":"Ruth","lastname":"Vance","age":32,"gender":"F","address":"229 Adams Street","employer":"Zilidium","email":"ruthvance@zilidium.com","city":"Allison","state":"IA","_is_real":true} +{"index":{"_id":"725"}} +{"account_number":725,"balance":14677,"firstname":"Reeves","lastname":"Tillman","age":26,"gender":"M","address":"674 Ivan Court","employer":"Cemention","email":"reevestillman@cemention.com","city":"Navarre","state":"MA","_is_real":true} +{"index":{"_id":"732"}} +{"account_number":732,"balance":38445,"firstname":"Delia","lastname":"Cruz","age":37,"gender":"F","address":"870 Cheever Place","employer":"Multron","email":"deliacruz@multron.com","city":"Cresaptown","state":"NH","_is_real":true} +{"index":{"_id":"737"}} +{"account_number":737,"balance":40431,"firstname":"Sampson","lastname":"Yates","age":23,"gender":"F","address":"214 Cox Place","employer":"Signidyne","email":"sampsonyates@signidyne.com","city":"Brazos","state":"GA","_is_real":true} +{"index":{"_id":"744"}} +{"account_number":744,"balance":8690,"firstname":"Bernard","lastname":"Martinez","age":21,"gender":"M","address":"148 Dunne Place","employer":"Dragbot","email":"bernardmartinez@dragbot.com","city":"Moraida","state":"MN","_is_real":true} +{"index":{"_id":"749"}} +{"account_number":749,"balance":1249,"firstname":"Rush","lastname":"Boyle","age":36,"gender":"M","address":"310 Argyle Road","employer":"Sportan","email":"rushboyle@sportan.com","city":"Brady","state":"WA","_is_real":true} +{"index":{"_id":"751"}} +{"account_number":751,"balance":49252,"firstname":"Patrick","lastname":"Osborne","age":23,"gender":"M","address":"915 Prospect Avenue","employer":"Gynko","email":"patrickosborne@gynko.com","city":"Takilma","state":"MO","_is_real":true} +{"index":{"_id":"756"}} +{"account_number":756,"balance":40006,"firstname":"Jasmine","lastname":"Howell","age":32,"gender":"M","address":"605 Elliott Walk","employer":"Ecratic","email":"jasminehowell@ecratic.com","city":"Harrodsburg","state":"OH","_is_real":true} +{"index":{"_id":"763"}} +{"account_number":763,"balance":12091,"firstname":"Liz","lastname":"Bentley","age":22,"gender":"F","address":"933 Debevoise Avenue","employer":"Nipaz","email":"lizbentley@nipaz.com","city":"Glenville","state":"NJ","_is_real":true} +{"index":{"_id":"768"}} +{"account_number":768,"balance":2213,"firstname":"Sondra","lastname":"Soto","age":21,"gender":"M","address":"625 Colonial Road","employer":"Navir","email":"sondrasoto@navir.com","city":"Benson","state":"VA","_is_real":true} +{"index":{"_id":"770"}} +{"account_number":770,"balance":39505,"firstname":"Joann","lastname":"Crane","age":26,"gender":"M","address":"798 Farragut Place","employer":"Lingoage","email":"joanncrane@lingoage.com","city":"Kirk","state":"MA","_is_real":true} +{"index":{"_id":"775"}} +{"account_number":775,"balance":27943,"firstname":"Wilson","lastname":"Merritt","age":33,"gender":"F","address":"288 Thornton Street","employer":"Geeky","email":"wilsonmerritt@geeky.com","city":"Holtville","state":"HI","_is_real":true} +{"index":{"_id":"782"}} +{"account_number":782,"balance":3960,"firstname":"Maldonado","lastname":"Craig","age":36,"gender":"F","address":"345 Myrtle Avenue","employer":"Zilencio","email":"maldonadocraig@zilencio.com","city":"Yukon","state":"ID","_is_real":true} +{"index":{"_id":"787"}} +{"account_number":787,"balance":11876,"firstname":"Harper","lastname":"Wynn","age":21,"gender":"F","address":"139 Oceanic Avenue","employer":"Interfind","email":"harperwynn@interfind.com","city":"Gerber","state":"ND","_is_real":true} +{"index":{"_id":"794"}} +{"account_number":794,"balance":16491,"firstname":"Walker","lastname":"Charles","age":32,"gender":"M","address":"215 Kenilworth Place","employer":"Orbin","email":"walkercharles@orbin.com","city":"Rivers","state":"WI","_is_real":true} +{"index":{"_id":"799"}} +{"account_number":799,"balance":2889,"firstname":"Myra","lastname":"Guerra","age":28,"gender":"F","address":"625 Dahlgreen Place","employer":"Digigene","email":"myraguerra@digigene.com","city":"Draper","state":"CA","_is_real":true} +{"index":{"_id":"802"}} +{"account_number":802,"balance":19630,"firstname":"Gracie","lastname":"Foreman","age":40,"gender":"F","address":"219 Kent Avenue","employer":"Supportal","email":"gracieforeman@supportal.com","city":"Westboro","state":"NH","_is_real":true} +{"index":{"_id":"807"}} +{"account_number":807,"balance":29206,"firstname":"Hatfield","lastname":"Lowe","age":23,"gender":"M","address":"499 Adler Place","employer":"Lovepad","email":"hatfieldlowe@lovepad.com","city":"Wiscon","state":"DC","_is_real":true} +{"index":{"_id":"814"}} +{"account_number":814,"balance":9838,"firstname":"Morse","lastname":"Mcbride","age":26,"gender":"F","address":"776 Calyer Street","employer":"Inear","email":"morsemcbride@inear.com","city":"Kingstowne","state":"ND","_is_real":true} +{"index":{"_id":"819"}} +{"account_number":819,"balance":3971,"firstname":"Karyn","lastname":"Medina","age":24,"gender":"F","address":"417 Utica Avenue","employer":"Qnekt","email":"karynmedina@qnekt.com","city":"Kerby","state":"WY","_is_real":true} +{"index":{"_id":"821"}} +{"account_number":821,"balance":33271,"firstname":"Trisha","lastname":"Blankenship","age":22,"gender":"M","address":"329 Jamaica Avenue","employer":"Chorizon","email":"trishablankenship@chorizon.com","city":"Sexton","state":"VT","_is_real":true} +{"index":{"_id":"826"}} +{"account_number":826,"balance":11548,"firstname":"Summers","lastname":"Vinson","age":22,"gender":"F","address":"742 Irwin Street","employer":"Globoil","email":"summersvinson@globoil.com","city":"Callaghan","state":"WY","_is_real":true} +{"index":{"_id":"833"}} +{"account_number":833,"balance":46154,"firstname":"Woodward","lastname":"Hood","age":22,"gender":"M","address":"398 Atkins Avenue","employer":"Zedalis","email":"woodwardhood@zedalis.com","city":"Stonybrook","state":"NE","_is_real":true} +{"index":{"_id":"838"}} +{"account_number":838,"balance":24629,"firstname":"Latonya","lastname":"Blake","age":37,"gender":"F","address":"531 Milton Street","employer":"Rugstars","email":"latonyablake@rugstars.com","city":"Tedrow","state":"WA","_is_real":true} +{"index":{"_id":"840"}} +{"account_number":840,"balance":39615,"firstname":"Boone","lastname":"Gomez","age":38,"gender":"M","address":"256 Hampton Place","employer":"Geekular","email":"boonegomez@geekular.com","city":"Westerville","state":"HI","_is_real":true} +{"index":{"_id":"845"}} +{"account_number":845,"balance":35422,"firstname":"Tracy","lastname":"Vaughn","age":39,"gender":"M","address":"645 Rockaway Parkway","employer":"Andryx","email":"tracyvaughn@andryx.com","city":"Wilmington","state":"ME","_is_real":true} +{"index":{"_id":"852"}} +{"account_number":852,"balance":6041,"firstname":"Allen","lastname":"Hammond","age":26,"gender":"M","address":"793 Essex Street","employer":"Tersanki","email":"allenhammond@tersanki.com","city":"Osmond","state":"NC","_is_real":true} +{"index":{"_id":"857"}} +{"account_number":857,"balance":39678,"firstname":"Alyce","lastname":"Douglas","age":23,"gender":"M","address":"326 Robert Street","employer":"Earbang","email":"alycedouglas@earbang.com","city":"Thornport","state":"GA","_is_real":true} +{"index":{"_id":"864"}} +{"account_number":864,"balance":21804,"firstname":"Duffy","lastname":"Anthony","age":23,"gender":"M","address":"582 Cooke Court","employer":"Schoolio","email":"duffyanthony@schoolio.com","city":"Brenton","state":"CO","_is_real":true} +{"index":{"_id":"869"}} +{"account_number":869,"balance":43544,"firstname":"Corinne","lastname":"Robbins","age":25,"gender":"F","address":"732 Quentin Road","employer":"Orbaxter","email":"corinnerobbins@orbaxter.com","city":"Roy","state":"TN","_is_real":true} +{"index":{"_id":"871"}} +{"account_number":871,"balance":35854,"firstname":"Norma","lastname":"Burt","age":32,"gender":"M","address":"934 Cyrus Avenue","employer":"Magnafone","email":"normaburt@magnafone.com","city":"Eden","state":"TN","_is_real":true} +{"index":{"_id":"876"}} +{"account_number":876,"balance":48568,"firstname":"Brady","lastname":"Glover","age":21,"gender":"F","address":"565 Oceanview Avenue","employer":"Comvex","email":"bradyglover@comvex.com","city":"Noblestown","state":"ID","_is_real":true} +{"index":{"_id":"883"}} +{"account_number":883,"balance":33679,"firstname":"Austin","lastname":"Jefferson","age":34,"gender":"M","address":"846 Lincoln Avenue","employer":"Polarax","email":"austinjefferson@polarax.com","city":"Savannah","state":"CT","_is_real":true} +{"index":{"_id":"888"}} +{"account_number":888,"balance":22277,"firstname":"Myrna","lastname":"Herman","age":39,"gender":"F","address":"649 Harwood Place","employer":"Enthaze","email":"myrnaherman@enthaze.com","city":"Idamay","state":"AR","_is_real":true} +{"index":{"_id":"890"}} +{"account_number":890,"balance":31198,"firstname":"Alvarado","lastname":"Pate","age":25,"gender":"M","address":"269 Ashland Place","employer":"Ovolo","email":"alvaradopate@ovolo.com","city":"Volta","state":"MI","_is_real":true} +{"index":{"_id":"895"}} +{"account_number":895,"balance":7327,"firstname":"Lara","lastname":"Mcdaniel","age":36,"gender":"M","address":"854 Willow Place","employer":"Acusage","email":"laramcdaniel@acusage.com","city":"Imperial","state":"NC","_is_real":true} +{"index":{"_id":"903"}} +{"account_number":903,"balance":10238,"firstname":"Wade","lastname":"Page","age":35,"gender":"F","address":"685 Waldorf Court","employer":"Eplosion","email":"wadepage@eplosion.com","city":"Welda","state":"AL","_is_real":true} +{"index":{"_id":"908"}} +{"account_number":908,"balance":45975,"firstname":"Mosley","lastname":"Holloway","age":31,"gender":"M","address":"929 Eldert Lane","employer":"Anivet","email":"mosleyholloway@anivet.com","city":"Biehle","state":"MS","_is_real":true} +{"index":{"_id":"910"}} +{"account_number":910,"balance":36831,"firstname":"Esmeralda","lastname":"James","age":23,"gender":"F","address":"535 High Street","employer":"Terrasys","email":"esmeraldajames@terrasys.com","city":"Dubois","state":"IN","_is_real":true} +{"index":{"_id":"915"}} +{"account_number":915,"balance":19816,"firstname":"Farrell","lastname":"French","age":35,"gender":"F","address":"126 McKibbin Street","employer":"Techmania","email":"farrellfrench@techmania.com","city":"Wescosville","state":"AL","_is_real":true} +{"index":{"_id":"922"}} +{"account_number":922,"balance":39347,"firstname":"Irwin","lastname":"Pugh","age":32,"gender":"M","address":"463 Shale Street","employer":"Idego","email":"irwinpugh@idego.com","city":"Ivanhoe","state":"ID","_is_real":true} +{"index":{"_id":"927"}} +{"account_number":927,"balance":19976,"firstname":"Jeanette","lastname":"Acevedo","age":26,"gender":"M","address":"694 Polhemus Place","employer":"Halap","email":"jeanetteacevedo@halap.com","city":"Harrison","state":"MO","_is_real":true} +{"index":{"_id":"934"}} +{"account_number":934,"balance":43987,"firstname":"Freida","lastname":"Daniels","age":34,"gender":"M","address":"448 Cove Lane","employer":"Vurbo","email":"freidadaniels@vurbo.com","city":"Snelling","state":"NJ","_is_real":true} +{"index":{"_id":"939"}} +{"account_number":939,"balance":31228,"firstname":"Hodges","lastname":"Massey","age":37,"gender":"F","address":"431 Dahl Court","employer":"Kegular","email":"hodgesmassey@kegular.com","city":"Katonah","state":"MD","_is_real":true} +{"index":{"_id":"941"}} +{"account_number":941,"balance":38796,"firstname":"Kim","lastname":"Moss","age":28,"gender":"F","address":"105 Onderdonk Avenue","employer":"Digirang","email":"kimmoss@digirang.com","city":"Centerville","state":"TX","_is_real":true} +{"index":{"_id":"946"}} +{"account_number":946,"balance":42794,"firstname":"Ina","lastname":"Obrien","age":36,"gender":"M","address":"339 Rewe Street","employer":"Eclipsent","email":"inaobrien@eclipsent.com","city":"Soham","state":"RI","_is_real":true} +{"index":{"_id":"953"}} +{"account_number":953,"balance":1110,"firstname":"Baxter","lastname":"Black","age":27,"gender":"M","address":"720 Stillwell Avenue","employer":"Uplinx","email":"baxterblack@uplinx.com","city":"Drummond","state":"MN","_is_real":true} +{"index":{"_id":"958"}} +{"account_number":958,"balance":32849,"firstname":"Brown","lastname":"Wilkins","age":40,"gender":"M","address":"686 Delmonico Place","employer":"Medesign","email":"brownwilkins@medesign.com","city":"Shelby","state":"WY","_is_real":true} +{"index":{"_id":"960"}} +{"account_number":960,"balance":2905,"firstname":"Curry","lastname":"Vargas","age":40,"gender":"M","address":"242 Blake Avenue","employer":"Pearlesex","email":"curryvargas@pearlesex.com","city":"Henrietta","state":"NH","_is_real":true} +{"index":{"_id":"965"}} +{"account_number":965,"balance":21882,"firstname":"Patrica","lastname":"Melton","age":28,"gender":"M","address":"141 Rodney Street","employer":"Flexigen","email":"patricamelton@flexigen.com","city":"Klagetoh","state":"MD","_is_real":true} +{"index":{"_id":"972"}} +{"account_number":972,"balance":24719,"firstname":"Leona","lastname":"Christian","age":26,"gender":"F","address":"900 Woodpoint Road","employer":"Extrawear","email":"leonachristian@extrawear.com","city":"Roderfield","state":"MA","_is_real":true} +{"index":{"_id":"977"}} +{"account_number":977,"balance":6744,"firstname":"Rodgers","lastname":"Mccray","age":21,"gender":"F","address":"612 Duryea Place","employer":"Papricut","email":"rodgersmccray@papricut.com","city":"Marenisco","state":"MD","_is_real":true} +{"index":{"_id":"984"}} +{"account_number":984,"balance":1904,"firstname":"Viola","lastname":"Crawford","age":35,"gender":"F","address":"354 Linwood Street","employer":"Ginkle","email":"violacrawford@ginkle.com","city":"Witmer","state":"AR","_is_real":true} +{"index":{"_id":"989"}} +{"account_number":989,"balance":48622,"firstname":"Franklin","lastname":"Frank","age":38,"gender":"M","address":"270 Carlton Avenue","employer":"Shopabout","email":"franklinfrank@shopabout.com","city":"Guthrie","state":"NC","_is_real":true} +{"index":{"_id":"991"}} +{"account_number":991,"balance":4239,"firstname":"Connie","lastname":"Berry","age":28,"gender":"F","address":"647 Gardner Avenue","employer":"Flumbo","email":"connieberry@flumbo.com","city":"Frierson","state":"MO","_is_real":true} +{"index":{"_id":"996"}} +{"account_number":996,"balance":17541,"firstname":"Andrews","lastname":"Herrera","age":30,"gender":"F","address":"570 Vandam Street","employer":"Klugger","email":"andrewsherrera@klugger.com","city":"Whitehaven","state":"MN","_is_real":true} +{"index":{"_id":"0"}} +{"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO","_is_real":true} +{"index":{"_id":"5"}} +{"account_number":5,"balance":29342,"firstname":"Leola","lastname":"Stewart","age":30,"gender":"F","address":"311 Elm Place","employer":"Diginetic","email":"leolastewart@diginetic.com","city":"Fairview","state":"NJ","_is_real":true} +{"index":{"_id":"12"}} +{"account_number":12,"balance":37055,"firstname":"Stafford","lastname":"Brock","age":20,"gender":"F","address":"296 Wythe Avenue","employer":"Uncorp","email":"staffordbrock@uncorp.com","city":"Bend","state":"AL","_is_real":true} +{"index":{"_id":"17"}} +{"account_number":17,"balance":7831,"firstname":"Bessie","lastname":"Orr","age":31,"gender":"F","address":"239 Hinsdale Street","employer":"Skyplex","email":"bessieorr@skyplex.com","city":"Graball","state":"FL","_is_real":true} +{"index":{"_id":"24"}} +{"account_number":24,"balance":44182,"firstname":"Wood","lastname":"Dale","age":39,"gender":"M","address":"582 Gelston Avenue","employer":"Besto","email":"wooddale@besto.com","city":"Juntura","state":"MI","_is_real":true} +{"index":{"_id":"29"}} +{"account_number":29,"balance":27323,"firstname":"Leah","lastname":"Santiago","age":33,"gender":"M","address":"193 Schenck Avenue","employer":"Isologix","email":"leahsantiago@isologix.com","city":"Gerton","state":"ND","_is_real":true} +{"index":{"_id":"31"}} +{"account_number":31,"balance":30443,"firstname":"Kristen","lastname":"Santana","age":22,"gender":"F","address":"130 Middagh Street","employer":"Dogspa","email":"kristensantana@dogspa.com","city":"Vale","state":"MA","_is_real":true} +{"index":{"_id":"36"}} +{"account_number":36,"balance":15902,"firstname":"Alexandra","lastname":"Nguyen","age":39,"gender":"F","address":"389 Elizabeth Place","employer":"Bittor","email":"alexandranguyen@bittor.com","city":"Hemlock","state":"KY","_is_real":true} +{"index":{"_id":"43"}} +{"account_number":43,"balance":33474,"firstname":"Ryan","lastname":"Howe","age":25,"gender":"M","address":"660 Huntington Street","employer":"Microluxe","email":"ryanhowe@microluxe.com","city":"Clara","state":"CT","_is_real":true} +{"index":{"_id":"48"}} +{"account_number":48,"balance":40608,"firstname":"Peck","lastname":"Downs","age":39,"gender":"F","address":"594 Dwight Street","employer":"Ramjob","email":"peckdowns@ramjob.com","city":"Coloma","state":"WA","_is_real":true} +{"index":{"_id":"50"}} +{"account_number":50,"balance":43695,"firstname":"Sheena","lastname":"Kirkland","age":33,"gender":"M","address":"598 Bank Street","employer":"Zerbina","email":"sheenakirkland@zerbina.com","city":"Walland","state":"IN","_is_real":true} +{"index":{"_id":"55"}} +{"account_number":55,"balance":22020,"firstname":"Shelia","lastname":"Puckett","age":33,"gender":"M","address":"265 Royce Place","employer":"Izzby","email":"sheliapuckett@izzby.com","city":"Slovan","state":"HI","_is_real":true} +{"index":{"_id":"62"}} +{"account_number":62,"balance":43065,"firstname":"Lester","lastname":"Stanton","age":37,"gender":"M","address":"969 Doughty Street","employer":"Geekko","email":"lesterstanton@geekko.com","city":"Itmann","state":"DC","_is_real":true} +{"index":{"_id":"67"}} +{"account_number":67,"balance":39430,"firstname":"Isabelle","lastname":"Spence","age":39,"gender":"M","address":"718 Troy Avenue","employer":"Geeketron","email":"isabellespence@geeketron.com","city":"Camptown","state":"WA","_is_real":true} +{"index":{"_id":"74"}} +{"account_number":74,"balance":47167,"firstname":"Lauri","lastname":"Saunders","age":38,"gender":"F","address":"768 Lynch Street","employer":"Securia","email":"laurisaunders@securia.com","city":"Caroline","state":"TN","_is_real":true} +{"index":{"_id":"79"}} +{"account_number":79,"balance":28185,"firstname":"Booker","lastname":"Lowery","age":29,"gender":"M","address":"817 Campus Road","employer":"Sensate","email":"bookerlowery@sensate.com","city":"Carlos","state":"MT","_is_real":true} +{"index":{"_id":"81"}} +{"account_number":81,"balance":46568,"firstname":"Dennis","lastname":"Gilbert","age":40,"gender":"M","address":"619 Minna Street","employer":"Melbacor","email":"dennisgilbert@melbacor.com","city":"Kersey","state":"ND","_is_real":true} +{"index":{"_id":"86"}} +{"account_number":86,"balance":15428,"firstname":"Walton","lastname":"Butler","age":36,"gender":"M","address":"999 Schenck Street","employer":"Unisure","email":"waltonbutler@unisure.com","city":"Bentonville","state":"IL","_is_real":true} +{"index":{"_id":"93"}} +{"account_number":93,"balance":17728,"firstname":"Jeri","lastname":"Booth","age":31,"gender":"M","address":"322 Roosevelt Court","employer":"Geekology","email":"jeribooth@geekology.com","city":"Leming","state":"ND","_is_real":true} +{"index":{"_id":"98"}} +{"account_number":98,"balance":15085,"firstname":"Cora","lastname":"Barrett","age":24,"gender":"F","address":"555 Neptune Court","employer":"Kiosk","email":"corabarrett@kiosk.com","city":"Independence","state":"MN","_is_real":true} +{"index":{"_id":"101"}} +{"account_number":101,"balance":43400,"firstname":"Cecelia","lastname":"Grimes","age":31,"gender":"M","address":"972 Lincoln Place","employer":"Ecosys","email":"ceceliagrimes@ecosys.com","city":"Manchester","state":"AR","_is_real":true} +{"index":{"_id":"106"}} +{"account_number":106,"balance":8212,"firstname":"Josefina","lastname":"Wagner","age":36,"gender":"M","address":"418 Estate Road","employer":"Kyaguru","email":"josefinawagner@kyaguru.com","city":"Darbydale","state":"FL","_is_real":true} +{"index":{"_id":"113"}} +{"account_number":113,"balance":41652,"firstname":"Burt","lastname":"Moses","age":27,"gender":"M","address":"633 Berry Street","employer":"Uni","email":"burtmoses@uni.com","city":"Russellville","state":"CT","_is_real":true} +{"index":{"_id":"118"}} +{"account_number":118,"balance":2223,"firstname":"Ballard","lastname":"Vasquez","age":33,"gender":"F","address":"101 Bush Street","employer":"Intergeek","email":"ballardvasquez@intergeek.com","city":"Century","state":"MN","_is_real":true} +{"index":{"_id":"120"}} +{"account_number":120,"balance":38565,"firstname":"Browning","lastname":"Rodriquez","age":33,"gender":"M","address":"910 Moore Street","employer":"Opportech","email":"browningrodriquez@opportech.com","city":"Cutter","state":"ND","_is_real":true} +{"index":{"_id":"125"}} +{"account_number":125,"balance":5396,"firstname":"Tanisha","lastname":"Dixon","age":30,"gender":"M","address":"482 Hancock Street","employer":"Junipoor","email":"tanishadixon@junipoor.com","city":"Wauhillau","state":"IA","_is_real":true} +{"index":{"_id":"132"}} +{"account_number":132,"balance":37707,"firstname":"Horton","lastname":"Romero","age":35,"gender":"M","address":"427 Rutherford Place","employer":"Affluex","email":"hortonromero@affluex.com","city":"Hall","state":"AK","_is_real":true} +{"index":{"_id":"137"}} +{"account_number":137,"balance":3596,"firstname":"Frost","lastname":"Freeman","age":29,"gender":"F","address":"191 Dennett Place","employer":"Beadzza","email":"frostfreeman@beadzza.com","city":"Sabillasville","state":"HI","_is_real":true} +{"index":{"_id":"144"}} +{"account_number":144,"balance":43257,"firstname":"Evans","lastname":"Dyer","age":30,"gender":"F","address":"912 Post Court","employer":"Magmina","email":"evansdyer@magmina.com","city":"Gordon","state":"HI","_is_real":true} +{"index":{"_id":"149"}} +{"account_number":149,"balance":22994,"firstname":"Megan","lastname":"Gonzales","age":21,"gender":"M","address":"836 Tampa Court","employer":"Andershun","email":"megangonzales@andershun.com","city":"Rockhill","state":"AL","_is_real":true} +{"index":{"_id":"151"}} +{"account_number":151,"balance":34473,"firstname":"Kent","lastname":"Joyner","age":20,"gender":"F","address":"799 Truxton Street","employer":"Kozgene","email":"kentjoyner@kozgene.com","city":"Allamuchy","state":"DC","_is_real":true} +{"index":{"_id":"156"}} +{"account_number":156,"balance":40185,"firstname":"Sloan","lastname":"Pennington","age":24,"gender":"F","address":"573 Opal Court","employer":"Hopeli","email":"sloanpennington@hopeli.com","city":"Evergreen","state":"CT","_is_real":true} +{"index":{"_id":"163"}} +{"account_number":163,"balance":43075,"firstname":"Wilda","lastname":"Norman","age":33,"gender":"F","address":"173 Beadel Street","employer":"Kog","email":"wildanorman@kog.com","city":"Bodega","state":"ME","_is_real":true} +{"index":{"_id":"168"}} +{"account_number":168,"balance":49568,"firstname":"Carissa","lastname":"Simon","age":20,"gender":"M","address":"975 Flatbush Avenue","employer":"Zillacom","email":"carissasimon@zillacom.com","city":"Neibert","state":"IL","_is_real":true} +{"index":{"_id":"170"}} +{"account_number":170,"balance":6025,"firstname":"Mann","lastname":"Madden","age":36,"gender":"F","address":"161 Radde Place","employer":"Farmex","email":"mannmadden@farmex.com","city":"Thermal","state":"LA","_is_real":true} +{"index":{"_id":"175"}} +{"account_number":175,"balance":16213,"firstname":"Montoya","lastname":"Donaldson","age":28,"gender":"F","address":"481 Morton Street","employer":"Envire","email":"montoyadonaldson@envire.com","city":"Delco","state":"MA","_is_real":true} +{"index":{"_id":"182"}} +{"account_number":182,"balance":7803,"firstname":"Manuela","lastname":"Dillon","age":21,"gender":"M","address":"742 Garnet Street","employer":"Moreganic","email":"manueladillon@moreganic.com","city":"Ilchester","state":"TX","_is_real":true} +{"index":{"_id":"187"}} +{"account_number":187,"balance":26581,"firstname":"Autumn","lastname":"Hodges","age":35,"gender":"M","address":"757 Granite Street","employer":"Ezentia","email":"autumnhodges@ezentia.com","city":"Martinsville","state":"KY","_is_real":true} +{"index":{"_id":"194"}} +{"account_number":194,"balance":16311,"firstname":"Beck","lastname":"Rosario","age":39,"gender":"M","address":"721 Cambridge Place","employer":"Zoid","email":"beckrosario@zoid.com","city":"Efland","state":"ID","_is_real":true} +{"index":{"_id":"199"}} +{"account_number":199,"balance":18086,"firstname":"Branch","lastname":"Love","age":26,"gender":"M","address":"458 Commercial Street","employer":"Frolix","email":"branchlove@frolix.com","city":"Caspar","state":"NC","_is_real":true} +{"index":{"_id":"202"}} +{"account_number":202,"balance":26466,"firstname":"Medina","lastname":"Brown","age":31,"gender":"F","address":"519 Sunnyside Court","employer":"Bleendot","email":"medinabrown@bleendot.com","city":"Winfred","state":"MI","_is_real":true} +{"index":{"_id":"207"}} +{"account_number":207,"balance":45535,"firstname":"Evelyn","lastname":"Lara","age":35,"gender":"F","address":"636 Chestnut Street","employer":"Ultrasure","email":"evelynlara@ultrasure.com","city":"Logan","state":"MI","_is_real":true} +{"index":{"_id":"214"}} +{"account_number":214,"balance":24418,"firstname":"Luann","lastname":"Faulkner","age":37,"gender":"F","address":"697 Hazel Court","employer":"Zolar","email":"luannfaulkner@zolar.com","city":"Ticonderoga","state":"TX","_is_real":true} +{"index":{"_id":"219"}} +{"account_number":219,"balance":17127,"firstname":"Edwards","lastname":"Hurley","age":25,"gender":"M","address":"834 Stockholm Street","employer":"Austech","email":"edwardshurley@austech.com","city":"Bayview","state":"NV","_is_real":true} +{"index":{"_id":"221"}} +{"account_number":221,"balance":15803,"firstname":"Benjamin","lastname":"Barrera","age":34,"gender":"M","address":"568 Main Street","employer":"Zaphire","email":"benjaminbarrera@zaphire.com","city":"Germanton","state":"WY","_is_real":true} +{"index":{"_id":"226"}} +{"account_number":226,"balance":37720,"firstname":"Wilkins","lastname":"Brady","age":40,"gender":"F","address":"486 Baltic Street","employer":"Dogtown","email":"wilkinsbrady@dogtown.com","city":"Condon","state":"MT","_is_real":true} +{"index":{"_id":"233"}} +{"account_number":233,"balance":23020,"firstname":"Washington","lastname":"Walsh","age":27,"gender":"M","address":"366 Church Avenue","employer":"Candecor","email":"washingtonwalsh@candecor.com","city":"Westphalia","state":"MA","_is_real":true} +{"index":{"_id":"238"}} +{"account_number":238,"balance":21287,"firstname":"Constance","lastname":"Wong","age":28,"gender":"M","address":"496 Brown Street","employer":"Grainspot","email":"constancewong@grainspot.com","city":"Cecilia","state":"IN","_is_real":true} +{"index":{"_id":"240"}} +{"account_number":240,"balance":49741,"firstname":"Oconnor","lastname":"Clay","age":35,"gender":"F","address":"659 Highland Boulevard","employer":"Franscene","email":"oconnorclay@franscene.com","city":"Kilbourne","state":"NH","_is_real":true} +{"index":{"_id":"245"}} +{"account_number":245,"balance":22026,"firstname":"Fran","lastname":"Bolton","age":28,"gender":"F","address":"147 Jerome Street","employer":"Solaren","email":"franbolton@solaren.com","city":"Nash","state":"RI","_is_real":true} +{"index":{"_id":"252"}} +{"account_number":252,"balance":18831,"firstname":"Elvia","lastname":"Poole","age":22,"gender":"F","address":"836 Delevan Street","employer":"Velity","email":"elviapoole@velity.com","city":"Groveville","state":"MI","_is_real":true} +{"index":{"_id":"257"}} +{"account_number":257,"balance":5318,"firstname":"Olive","lastname":"Oneil","age":35,"gender":"F","address":"457 Decatur Street","employer":"Helixo","email":"oliveoneil@helixo.com","city":"Chicopee","state":"MI","_is_real":true} +{"index":{"_id":"264"}} +{"account_number":264,"balance":22084,"firstname":"Samantha","lastname":"Ferrell","age":35,"gender":"F","address":"488 Fulton Street","employer":"Flum","email":"samanthaferrell@flum.com","city":"Brandywine","state":"MT","_is_real":true} +{"index":{"_id":"269"}} +{"account_number":269,"balance":43317,"firstname":"Crosby","lastname":"Figueroa","age":34,"gender":"M","address":"910 Aurelia Court","employer":"Pyramia","email":"crosbyfigueroa@pyramia.com","city":"Leyner","state":"OH","_is_real":true} +{"index":{"_id":"271"}} +{"account_number":271,"balance":11864,"firstname":"Holt","lastname":"Walter","age":30,"gender":"F","address":"645 Poplar Avenue","employer":"Grupoli","email":"holtwalter@grupoli.com","city":"Mansfield","state":"OR","_is_real":true} +{"index":{"_id":"276"}} +{"account_number":276,"balance":11606,"firstname":"Pittman","lastname":"Mathis","age":23,"gender":"F","address":"567 Charles Place","employer":"Zuvy","email":"pittmanmathis@zuvy.com","city":"Roeville","state":"KY","_is_real":true} +{"index":{"_id":"283"}} +{"account_number":283,"balance":24070,"firstname":"Fuentes","lastname":"Foley","age":30,"gender":"M","address":"729 Walker Court","employer":"Knowlysis","email":"fuentesfoley@knowlysis.com","city":"Tryon","state":"TN","_is_real":true} +{"index":{"_id":"288"}} +{"account_number":288,"balance":27243,"firstname":"Wong","lastname":"Stone","age":39,"gender":"F","address":"440 Willoughby Street","employer":"Zentix","email":"wongstone@zentix.com","city":"Wheatfields","state":"DC","_is_real":true} +{"index":{"_id":"290"}} +{"account_number":290,"balance":26103,"firstname":"Neva","lastname":"Burgess","age":37,"gender":"F","address":"985 Wyona Street","employer":"Slofast","email":"nevaburgess@slofast.com","city":"Cawood","state":"DC","_is_real":true} +{"index":{"_id":"295"}} +{"account_number":295,"balance":37358,"firstname":"Howe","lastname":"Nash","age":20,"gender":"M","address":"833 Union Avenue","employer":"Aquacine","email":"howenash@aquacine.com","city":"Indio","state":"MN","_is_real":true} +{"index":{"_id":"303"}} +{"account_number":303,"balance":21976,"firstname":"Huffman","lastname":"Green","age":24,"gender":"F","address":"455 Colby Court","employer":"Comtest","email":"huffmangreen@comtest.com","city":"Weeksville","state":"UT","_is_real":true} +{"index":{"_id":"308"}} +{"account_number":308,"balance":33989,"firstname":"Glass","lastname":"Schroeder","age":25,"gender":"F","address":"670 Veterans Avenue","employer":"Realmo","email":"glassschroeder@realmo.com","city":"Gratton","state":"NY","_is_real":true} +{"index":{"_id":"310"}} +{"account_number":310,"balance":23049,"firstname":"Shannon","lastname":"Morton","age":39,"gender":"F","address":"412 Pleasant Place","employer":"Ovation","email":"shannonmorton@ovation.com","city":"Edgar","state":"AZ","_is_real":true} +{"index":{"_id":"315"}} +{"account_number":315,"balance":1314,"firstname":"Clare","lastname":"Morrow","age":33,"gender":"F","address":"728 Madeline Court","employer":"Gaptec","email":"claremorrow@gaptec.com","city":"Mapletown","state":"PA","_is_real":true} +{"index":{"_id":"322"}} +{"account_number":322,"balance":6303,"firstname":"Gilliam","lastname":"Horne","age":27,"gender":"M","address":"414 Florence Avenue","employer":"Shepard","email":"gilliamhorne@shepard.com","city":"Winesburg","state":"WY","_is_real":true} +{"index":{"_id":"327"}} +{"account_number":327,"balance":29294,"firstname":"Nell","lastname":"Contreras","age":27,"gender":"M","address":"694 Gold Street","employer":"Momentia","email":"nellcontreras@momentia.com","city":"Cumminsville","state":"AL","_is_real":true} +{"index":{"_id":"334"}} +{"account_number":334,"balance":9178,"firstname":"Cross","lastname":"Floyd","age":21,"gender":"F","address":"815 Herkimer Court","employer":"Maroptic","email":"crossfloyd@maroptic.com","city":"Kraemer","state":"AK","_is_real":true} +{"index":{"_id":"339"}} +{"account_number":339,"balance":3992,"firstname":"Franco","lastname":"Welch","age":38,"gender":"F","address":"776 Brightwater Court","employer":"Earthplex","email":"francowelch@earthplex.com","city":"Naomi","state":"ME","_is_real":true} +{"index":{"_id":"341"}} +{"account_number":341,"balance":44367,"firstname":"Alberta","lastname":"Bradford","age":30,"gender":"F","address":"670 Grant Avenue","employer":"Bugsall","email":"albertabradford@bugsall.com","city":"Romeville","state":"MT","_is_real":true} +{"index":{"_id":"346"}} +{"account_number":346,"balance":26594,"firstname":"Shelby","lastname":"Sanchez","age":36,"gender":"F","address":"257 Fillmore Avenue","employer":"Geekus","email":"shelbysanchez@geekus.com","city":"Seymour","state":"CO","_is_real":true} +{"index":{"_id":"353"}} +{"account_number":353,"balance":45182,"firstname":"Rivera","lastname":"Sherman","age":37,"gender":"M","address":"603 Garden Place","employer":"Bovis","email":"riverasherman@bovis.com","city":"Otranto","state":"CA","_is_real":true} +{"index":{"_id":"358"}} +{"account_number":358,"balance":44043,"firstname":"Hale","lastname":"Baldwin","age":40,"gender":"F","address":"845 Menahan Street","employer":"Kidgrease","email":"halebaldwin@kidgrease.com","city":"Day","state":"AK","_is_real":true} +{"index":{"_id":"360"}} +{"account_number":360,"balance":26651,"firstname":"Ward","lastname":"Hicks","age":34,"gender":"F","address":"592 Brighton Court","employer":"Biotica","email":"wardhicks@biotica.com","city":"Kanauga","state":"VT","_is_real":true} +{"index":{"_id":"365"}} +{"account_number":365,"balance":3176,"firstname":"Sanders","lastname":"Holder","age":31,"gender":"F","address":"453 Cypress Court","employer":"Geekola","email":"sandersholder@geekola.com","city":"Staples","state":"TN","_is_real":true} +{"index":{"_id":"372"}} +{"account_number":372,"balance":28566,"firstname":"Alba","lastname":"Forbes","age":24,"gender":"M","address":"814 Meserole Avenue","employer":"Isostream","email":"albaforbes@isostream.com","city":"Clarence","state":"OR","_is_real":true} +{"index":{"_id":"377"}} +{"account_number":377,"balance":5374,"firstname":"Margo","lastname":"Gay","age":34,"gender":"F","address":"613 Chase Court","employer":"Rotodyne","email":"margogay@rotodyne.com","city":"Waumandee","state":"KS","_is_real":true} +{"index":{"_id":"384"}} +{"account_number":384,"balance":48758,"firstname":"Sallie","lastname":"Houston","age":31,"gender":"F","address":"836 Polar Street","employer":"Squish","email":"salliehouston@squish.com","city":"Morningside","state":"NC","_is_real":true} +{"index":{"_id":"389"}} +{"account_number":389,"balance":8839,"firstname":"York","lastname":"Cummings","age":27,"gender":"M","address":"778 Centre Street","employer":"Insurity","email":"yorkcummings@insurity.com","city":"Freeburn","state":"RI","_is_real":true} +{"index":{"_id":"391"}} +{"account_number":391,"balance":14733,"firstname":"Holman","lastname":"Jordan","age":30,"gender":"M","address":"391 Forrest Street","employer":"Maineland","email":"holmanjordan@maineland.com","city":"Cade","state":"CT","_is_real":true} +{"index":{"_id":"396"}} +{"account_number":396,"balance":14613,"firstname":"Marsha","lastname":"Elliott","age":38,"gender":"F","address":"297 Liberty Avenue","employer":"Orbiflex","email":"marshaelliott@orbiflex.com","city":"Windsor","state":"TX","_is_real":true} +{"index":{"_id":"404"}} +{"account_number":404,"balance":34978,"firstname":"Massey","lastname":"Becker","age":26,"gender":"F","address":"930 Pitkin Avenue","employer":"Genekom","email":"masseybecker@genekom.com","city":"Blairstown","state":"OR","_is_real":true} +{"index":{"_id":"409"}} +{"account_number":409,"balance":36960,"firstname":"Maura","lastname":"Glenn","age":31,"gender":"M","address":"183 Poly Place","employer":"Viagreat","email":"mauraglenn@viagreat.com","city":"Foscoe","state":"DE","_is_real":true} +{"index":{"_id":"411"}} +{"account_number":411,"balance":1172,"firstname":"Guzman","lastname":"Whitfield","age":22,"gender":"M","address":"181 Perry Terrace","employer":"Springbee","email":"guzmanwhitfield@springbee.com","city":"Balm","state":"IN","_is_real":true} +{"index":{"_id":"416"}} +{"account_number":416,"balance":27169,"firstname":"Hunt","lastname":"Schwartz","age":28,"gender":"F","address":"461 Havens Place","employer":"Danja","email":"huntschwartz@danja.com","city":"Grenelefe","state":"NV","_is_real":true} +{"index":{"_id":"423"}} +{"account_number":423,"balance":38852,"firstname":"Hines","lastname":"Underwood","age":21,"gender":"F","address":"284 Louise Terrace","employer":"Namegen","email":"hinesunderwood@namegen.com","city":"Downsville","state":"CO","_is_real":true} +{"index":{"_id":"428"}} +{"account_number":428,"balance":13925,"firstname":"Stephens","lastname":"Cain","age":20,"gender":"F","address":"189 Summit Street","employer":"Rocklogic","email":"stephenscain@rocklogic.com","city":"Bourg","state":"HI","_is_real":true} +{"index":{"_id":"430"}} +{"account_number":430,"balance":15251,"firstname":"Alejandra","lastname":"Chavez","age":34,"gender":"M","address":"651 Butler Place","employer":"Gology","email":"alejandrachavez@gology.com","city":"Allensworth","state":"VT","_is_real":true} +{"index":{"_id":"435"}} +{"account_number":435,"balance":14654,"firstname":"Sue","lastname":"Lopez","age":22,"gender":"F","address":"632 Stone Avenue","employer":"Emergent","email":"suelopez@emergent.com","city":"Waterford","state":"TN","_is_real":true} +{"index":{"_id":"442"}} +{"account_number":442,"balance":36211,"firstname":"Lawanda","lastname":"Leon","age":27,"gender":"F","address":"126 Canal Avenue","employer":"Xixan","email":"lawandaleon@xixan.com","city":"Berwind","state":"TN","_is_real":true} +{"index":{"_id":"447"}} +{"account_number":447,"balance":11402,"firstname":"Lucia","lastname":"Livingston","age":35,"gender":"M","address":"773 Lake Avenue","employer":"Soprano","email":"lucialivingston@soprano.com","city":"Edgewater","state":"TN","_is_real":true} +{"index":{"_id":"454"}} +{"account_number":454,"balance":31687,"firstname":"Alicia","lastname":"Rollins","age":22,"gender":"F","address":"483 Verona Place","employer":"Boilcat","email":"aliciarollins@boilcat.com","city":"Lutsen","state":"MD","_is_real":true} +{"index":{"_id":"459"}} +{"account_number":459,"balance":18869,"firstname":"Pamela","lastname":"Henry","age":20,"gender":"F","address":"361 Locust Avenue","employer":"Imageflow","email":"pamelahenry@imageflow.com","city":"Greenfields","state":"OH","_is_real":true} +{"index":{"_id":"461"}} +{"account_number":461,"balance":38807,"firstname":"Mcbride","lastname":"Padilla","age":34,"gender":"F","address":"550 Borinquen Pl","employer":"Zepitope","email":"mcbridepadilla@zepitope.com","city":"Emory","state":"AZ","_is_real":true} +{"index":{"_id":"466"}} +{"account_number":466,"balance":25109,"firstname":"Marcie","lastname":"Mcmillan","age":30,"gender":"F","address":"947 Gain Court","employer":"Entroflex","email":"marciemcmillan@entroflex.com","city":"Ronco","state":"ND","_is_real":true} +{"index":{"_id":"473"}} +{"account_number":473,"balance":5391,"firstname":"Susan","lastname":"Luna","age":25,"gender":"F","address":"521 Bogart Street","employer":"Zaya","email":"susanluna@zaya.com","city":"Grazierville","state":"MI","_is_real":true} +{"index":{"_id":"478"}} +{"account_number":478,"balance":28044,"firstname":"Dana","lastname":"Decker","age":35,"gender":"M","address":"627 Dobbin Street","employer":"Acrodance","email":"danadecker@acrodance.com","city":"Sharon","state":"MN","_is_real":true} +{"index":{"_id":"480"}} +{"account_number":480,"balance":40807,"firstname":"Anastasia","lastname":"Parker","age":24,"gender":"M","address":"650 Folsom Place","employer":"Zilladyne","email":"anastasiaparker@zilladyne.com","city":"Oberlin","state":"WY","_is_real":true} +{"index":{"_id":"485"}} +{"account_number":485,"balance":44235,"firstname":"Albert","lastname":"Roberts","age":40,"gender":"M","address":"385 Harman Street","employer":"Stralum","email":"albertroberts@stralum.com","city":"Watrous","state":"NM","_is_real":true} +{"index":{"_id":"492"}} +{"account_number":492,"balance":31055,"firstname":"Burnett","lastname":"Briggs","age":35,"gender":"M","address":"987 Cass Place","employer":"Pharmex","email":"burnettbriggs@pharmex.com","city":"Cornfields","state":"TX","_is_real":true} +{"index":{"_id":"497"}} +{"account_number":497,"balance":13493,"firstname":"Doyle","lastname":"Jenkins","age":30,"gender":"M","address":"205 Nevins Street","employer":"Unia","email":"doylejenkins@unia.com","city":"Nicut","state":"DC","_is_real":true} +{"index":{"_id":"500"}} +{"account_number":500,"balance":39143,"firstname":"Pope","lastname":"Keith","age":28,"gender":"F","address":"537 Fane Court","employer":"Zboo","email":"popekeith@zboo.com","city":"Courtland","state":"AL","_is_real":true} +{"index":{"_id":"505"}} +{"account_number":505,"balance":45493,"firstname":"Shelley","lastname":"Webb","age":29,"gender":"M","address":"873 Crawford Avenue","employer":"Quadeebo","email":"shelleywebb@quadeebo.com","city":"Topanga","state":"IL","_is_real":true} +{"index":{"_id":"512"}} +{"account_number":512,"balance":47432,"firstname":"Alisha","lastname":"Morales","age":29,"gender":"M","address":"623 Batchelder Street","employer":"Terragen","email":"alishamorales@terragen.com","city":"Gilmore","state":"VA","_is_real":true} +{"index":{"_id":"517"}} +{"account_number":517,"balance":3022,"firstname":"Allyson","lastname":"Walls","age":38,"gender":"F","address":"334 Coffey Street","employer":"Gorganic","email":"allysonwalls@gorganic.com","city":"Dahlen","state":"GA","_is_real":true} +{"index":{"_id":"524"}} +{"account_number":524,"balance":49334,"firstname":"Salas","lastname":"Farley","age":30,"gender":"F","address":"499 Trucklemans Lane","employer":"Xumonk","email":"salasfarley@xumonk.com","city":"Noxen","state":"AL","_is_real":true} +{"index":{"_id":"529"}} +{"account_number":529,"balance":21788,"firstname":"Deann","lastname":"Fisher","age":23,"gender":"F","address":"511 Buffalo Avenue","employer":"Twiist","email":"deannfisher@twiist.com","city":"Templeton","state":"WA","_is_real":true} +{"index":{"_id":"531"}} +{"account_number":531,"balance":39770,"firstname":"Janet","lastname":"Pena","age":38,"gender":"M","address":"645 Livonia Avenue","employer":"Corecom","email":"janetpena@corecom.com","city":"Garberville","state":"OK","_is_real":true} +{"index":{"_id":"536"}} +{"account_number":536,"balance":6255,"firstname":"Emma","lastname":"Adkins","age":33,"gender":"F","address":"971 Calder Place","employer":"Ontagene","email":"emmaadkins@ontagene.com","city":"Ruckersville","state":"GA","_is_real":true} +{"index":{"_id":"543"}} +{"account_number":543,"balance":48022,"firstname":"Marina","lastname":"Rasmussen","age":31,"gender":"M","address":"446 Love Lane","employer":"Crustatia","email":"marinarasmussen@crustatia.com","city":"Statenville","state":"MD","_is_real":true} +{"index":{"_id":"548"}} +{"account_number":548,"balance":36930,"firstname":"Sandra","lastname":"Andrews","age":37,"gender":"M","address":"973 Prospect Street","employer":"Datagene","email":"sandraandrews@datagene.com","city":"Inkerman","state":"MO","_is_real":true} +{"index":{"_id":"550"}} +{"account_number":550,"balance":32238,"firstname":"Walsh","lastname":"Goodwin","age":22,"gender":"M","address":"953 Canda Avenue","employer":"Proflex","email":"walshgoodwin@proflex.com","city":"Ypsilanti","state":"MT","_is_real":true} +{"index":{"_id":"555"}} +{"account_number":555,"balance":10750,"firstname":"Fannie","lastname":"Slater","age":31,"gender":"M","address":"457 Tech Place","employer":"Kineticut","email":"fannieslater@kineticut.com","city":"Basye","state":"MO","_is_real":true} +{"index":{"_id":"562"}} +{"account_number":562,"balance":10737,"firstname":"Sarah","lastname":"Strong","age":39,"gender":"F","address":"177 Pioneer Street","employer":"Megall","email":"sarahstrong@megall.com","city":"Ladera","state":"WY","_is_real":true} +{"index":{"_id":"567"}} +{"account_number":567,"balance":6507,"firstname":"Diana","lastname":"Dominguez","age":40,"gender":"M","address":"419 Albany Avenue","employer":"Ohmnet","email":"dianadominguez@ohmnet.com","city":"Wildwood","state":"TX","_is_real":true} +{"index":{"_id":"574"}} +{"account_number":574,"balance":32954,"firstname":"Andrea","lastname":"Mosley","age":24,"gender":"M","address":"368 Throop Avenue","employer":"Musix","email":"andreamosley@musix.com","city":"Blende","state":"DC","_is_real":true} +{"index":{"_id":"579"}} +{"account_number":579,"balance":12044,"firstname":"Banks","lastname":"Sawyer","age":36,"gender":"M","address":"652 Doone Court","employer":"Rooforia","email":"bankssawyer@rooforia.com","city":"Foxworth","state":"ND","_is_real":true} +{"index":{"_id":"581"}} +{"account_number":581,"balance":16525,"firstname":"Fuller","lastname":"Mcintyre","age":32,"gender":"M","address":"169 Bergen Place","employer":"Applideck","email":"fullermcintyre@applideck.com","city":"Kenvil","state":"NY","_is_real":true} +{"index":{"_id":"586"}} +{"account_number":586,"balance":13644,"firstname":"Love","lastname":"Velasquez","age":26,"gender":"F","address":"290 Girard Street","employer":"Zomboid","email":"lovevelasquez@zomboid.com","city":"Villarreal","state":"SD","_is_real":true} +{"index":{"_id":"593"}} +{"account_number":593,"balance":41230,"firstname":"Muriel","lastname":"Vazquez","age":37,"gender":"M","address":"395 Montgomery Street","employer":"Sustenza","email":"murielvazquez@sustenza.com","city":"Strykersville","state":"OK","_is_real":true} +{"index":{"_id":"598"}} +{"account_number":598,"balance":33251,"firstname":"Morgan","lastname":"Coleman","age":33,"gender":"M","address":"324 McClancy Place","employer":"Aclima","email":"morgancoleman@aclima.com","city":"Bowden","state":"WA","_is_real":true} +{"index":{"_id":"601"}} +{"account_number":601,"balance":20796,"firstname":"Vickie","lastname":"Valentine","age":34,"gender":"F","address":"432 Bassett Avenue","employer":"Comvene","email":"vickievalentine@comvene.com","city":"Teasdale","state":"UT","_is_real":true} +{"index":{"_id":"606"}} +{"account_number":606,"balance":28770,"firstname":"Michael","lastname":"Bray","age":31,"gender":"M","address":"935 Lake Place","employer":"Telepark","email":"michaelbray@telepark.com","city":"Lemoyne","state":"CT","_is_real":true} +{"index":{"_id":"613"}} +{"account_number":613,"balance":39340,"firstname":"Eddie","lastname":"Mccarty","age":34,"gender":"F","address":"971 Richards Street","employer":"Bisba","email":"eddiemccarty@bisba.com","city":"Fruitdale","state":"NY","_is_real":true} +{"index":{"_id":"618"}} +{"account_number":618,"balance":8976,"firstname":"Cheri","lastname":"Ford","age":30,"gender":"F","address":"803 Ridgewood Avenue","employer":"Zorromop","email":"cheriford@zorromop.com","city":"Gambrills","state":"VT","_is_real":true} +{"index":{"_id":"620"}} +{"account_number":620,"balance":7224,"firstname":"Coleen","lastname":"Bartlett","age":38,"gender":"M","address":"761 Carroll Street","employer":"Idealis","email":"coleenbartlett@idealis.com","city":"Mathews","state":"DE","_is_real":true} +{"index":{"_id":"625"}} +{"account_number":625,"balance":46010,"firstname":"Cynthia","lastname":"Johnston","age":23,"gender":"M","address":"142 Box Street","employer":"Zentry","email":"cynthiajohnston@zentry.com","city":"Makena","state":"MA","_is_real":true} +{"index":{"_id":"632"}} +{"account_number":632,"balance":40470,"firstname":"Kay","lastname":"Warren","age":20,"gender":"F","address":"422 Alabama Avenue","employer":"Realysis","email":"kaywarren@realysis.com","city":"Homestead","state":"HI","_is_real":true} +{"index":{"_id":"637"}} +{"account_number":637,"balance":3169,"firstname":"Kathy","lastname":"Carter","age":27,"gender":"F","address":"410 Jamison Lane","employer":"Limage","email":"kathycarter@limage.com","city":"Ernstville","state":"WA","_is_real":true} +{"index":{"_id":"644"}} +{"account_number":644,"balance":44021,"firstname":"Etta","lastname":"Miller","age":21,"gender":"F","address":"376 Lawton Street","employer":"Bluegrain","email":"ettamiller@bluegrain.com","city":"Baker","state":"MD","_is_real":true} +{"index":{"_id":"649"}} +{"account_number":649,"balance":20275,"firstname":"Jeanine","lastname":"Malone","age":26,"gender":"F","address":"114 Dodworth Street","employer":"Nixelt","email":"jeaninemalone@nixelt.com","city":"Keyport","state":"AK","_is_real":true} +{"index":{"_id":"651"}} +{"account_number":651,"balance":18360,"firstname":"Young","lastname":"Reeves","age":34,"gender":"M","address":"581 Plaza Street","employer":"Krog","email":"youngreeves@krog.com","city":"Sussex","state":"WY","_is_real":true} +{"index":{"_id":"656"}} +{"account_number":656,"balance":21632,"firstname":"Olson","lastname":"Hunt","age":36,"gender":"M","address":"342 Jaffray Street","employer":"Volax","email":"olsonhunt@volax.com","city":"Bangor","state":"WA","_is_real":true} +{"index":{"_id":"663"}} +{"account_number":663,"balance":2456,"firstname":"Rollins","lastname":"Richards","age":37,"gender":"M","address":"129 Sullivan Place","employer":"Geostele","email":"rollinsrichards@geostele.com","city":"Morgandale","state":"FL","_is_real":true} +{"index":{"_id":"668"}} +{"account_number":668,"balance":45069,"firstname":"Potter","lastname":"Michael","age":27,"gender":"M","address":"803 Glenmore Avenue","employer":"Ontality","email":"pottermichael@ontality.com","city":"Newkirk","state":"KS","_is_real":true} +{"index":{"_id":"670"}} +{"account_number":670,"balance":10178,"firstname":"Ollie","lastname":"Riley","age":22,"gender":"M","address":"252 Jackson Place","employer":"Adornica","email":"ollieriley@adornica.com","city":"Brethren","state":"WI","_is_real":true} +{"index":{"_id":"675"}} +{"account_number":675,"balance":36102,"firstname":"Fisher","lastname":"Shepard","age":27,"gender":"F","address":"859 Varick Street","employer":"Qot","email":"fishershepard@qot.com","city":"Diaperville","state":"MD","_is_real":true} +{"index":{"_id":"682"}} +{"account_number":682,"balance":14168,"firstname":"Anne","lastname":"Hale","age":22,"gender":"F","address":"708 Anthony Street","employer":"Cytrek","email":"annehale@cytrek.com","city":"Beechmont","state":"WV","_is_real":true} +{"index":{"_id":"687"}} +{"account_number":687,"balance":48630,"firstname":"Caroline","lastname":"Cox","age":31,"gender":"M","address":"626 Hillel Place","employer":"Opticon","email":"carolinecox@opticon.com","city":"Loma","state":"ND","_is_real":true} +{"index":{"_id":"694"}} +{"account_number":694,"balance":33125,"firstname":"Craig","lastname":"Palmer","age":31,"gender":"F","address":"273 Montrose Avenue","employer":"Comvey","email":"craigpalmer@comvey.com","city":"Cleary","state":"OK","_is_real":true} +{"index":{"_id":"699"}} +{"account_number":699,"balance":4156,"firstname":"Gallagher","lastname":"Marshall","age":37,"gender":"F","address":"648 Clifford Place","employer":"Exiand","email":"gallaghermarshall@exiand.com","city":"Belfair","state":"KY","_is_real":true} +{"index":{"_id":"702"}} +{"account_number":702,"balance":46490,"firstname":"Meadows","lastname":"Delgado","age":26,"gender":"M","address":"612 Jardine Place","employer":"Daisu","email":"meadowsdelgado@daisu.com","city":"Venice","state":"AR","_is_real":true} +{"index":{"_id":"707"}} +{"account_number":707,"balance":30325,"firstname":"Sonya","lastname":"Trevino","age":30,"gender":"F","address":"181 Irving Place","employer":"Atgen","email":"sonyatrevino@atgen.com","city":"Enetai","state":"TN","_is_real":true} +{"index":{"_id":"714"}} +{"account_number":714,"balance":16602,"firstname":"Socorro","lastname":"Murray","age":34,"gender":"F","address":"810 Manhattan Court","employer":"Isoswitch","email":"socorromurray@isoswitch.com","city":"Jugtown","state":"AZ","_is_real":true} +{"index":{"_id":"719"}} +{"account_number":719,"balance":33107,"firstname":"Leanna","lastname":"Reed","age":25,"gender":"F","address":"528 Krier Place","employer":"Rodeology","email":"leannareed@rodeology.com","city":"Carrizo","state":"WI","_is_real":true} +{"index":{"_id":"721"}} +{"account_number":721,"balance":32958,"firstname":"Mara","lastname":"Dickson","age":26,"gender":"M","address":"810 Harrison Avenue","employer":"Comtours","email":"maradickson@comtours.com","city":"Thynedale","state":"DE","_is_real":true} +{"index":{"_id":"726"}} +{"account_number":726,"balance":44737,"firstname":"Rosemary","lastname":"Salazar","age":21,"gender":"M","address":"290 Croton Loop","employer":"Rockabye","email":"rosemarysalazar@rockabye.com","city":"Helen","state":"IA","_is_real":true} +{"index":{"_id":"733"}} +{"account_number":733,"balance":15722,"firstname":"Lakeisha","lastname":"Mccarthy","age":37,"gender":"M","address":"782 Turnbull Avenue","employer":"Exosis","email":"lakeishamccarthy@exosis.com","city":"Caberfae","state":"NM","_is_real":true} +{"index":{"_id":"738"}} +{"account_number":738,"balance":44936,"firstname":"Rosalind","lastname":"Hunter","age":32,"gender":"M","address":"644 Eaton Court","employer":"Zolarity","email":"rosalindhunter@zolarity.com","city":"Cataract","state":"SD","_is_real":true} +{"index":{"_id":"740"}} +{"account_number":740,"balance":6143,"firstname":"Chambers","lastname":"Hahn","age":22,"gender":"M","address":"937 Windsor Place","employer":"Medalert","email":"chambershahn@medalert.com","city":"Dorneyville","state":"DC","_is_real":true} +{"index":{"_id":"745"}} +{"account_number":745,"balance":4572,"firstname":"Jacobs","lastname":"Sweeney","age":32,"gender":"M","address":"189 Lott Place","employer":"Comtent","email":"jacobssweeney@comtent.com","city":"Advance","state":"NJ","_is_real":true} +{"index":{"_id":"752"}} +{"account_number":752,"balance":14039,"firstname":"Jerry","lastname":"Rush","age":31,"gender":"M","address":"632 Dank Court","employer":"Ebidco","email":"jerryrush@ebidco.com","city":"Geyserville","state":"AR","_is_real":true} +{"index":{"_id":"757"}} +{"account_number":757,"balance":34628,"firstname":"Mccullough","lastname":"Moore","age":30,"gender":"F","address":"304 Hastings Street","employer":"Nikuda","email":"mcculloughmoore@nikuda.com","city":"Charco","state":"DC","_is_real":true} +{"index":{"_id":"764"}} +{"account_number":764,"balance":3728,"firstname":"Noemi","lastname":"Gill","age":30,"gender":"M","address":"427 Chester Street","employer":"Avit","email":"noemigill@avit.com","city":"Chesterfield","state":"AL","_is_real":true} +{"index":{"_id":"769"}} +{"account_number":769,"balance":15362,"firstname":"Francis","lastname":"Beck","age":28,"gender":"M","address":"454 Livingston Street","employer":"Furnafix","email":"francisbeck@furnafix.com","city":"Dunnavant","state":"HI","_is_real":true} +{"index":{"_id":"771"}} +{"account_number":771,"balance":32784,"firstname":"Jocelyn","lastname":"Boone","age":23,"gender":"M","address":"513 Division Avenue","employer":"Collaire","email":"jocelynboone@collaire.com","city":"Lisco","state":"VT","_is_real":true} +{"index":{"_id":"776"}} +{"account_number":776,"balance":29177,"firstname":"Duke","lastname":"Atkinson","age":24,"gender":"M","address":"520 Doscher Street","employer":"Tripsch","email":"dukeatkinson@tripsch.com","city":"Lafferty","state":"NC","_is_real":true} +{"index":{"_id":"783"}} +{"account_number":783,"balance":11911,"firstname":"Faith","lastname":"Cooper","age":25,"gender":"F","address":"539 Rapelye Street","employer":"Insuron","email":"faithcooper@insuron.com","city":"Jennings","state":"MN","_is_real":true} +{"index":{"_id":"788"}} +{"account_number":788,"balance":12473,"firstname":"Marianne","lastname":"Aguilar","age":39,"gender":"F","address":"213 Holly Street","employer":"Marqet","email":"marianneaguilar@marqet.com","city":"Alfarata","state":"HI","_is_real":true} +{"index":{"_id":"790"}} +{"account_number":790,"balance":29912,"firstname":"Ellis","lastname":"Sullivan","age":39,"gender":"F","address":"877 Coyle Street","employer":"Enersave","email":"ellissullivan@enersave.com","city":"Canby","state":"MS","_is_real":true} +{"index":{"_id":"795"}} +{"account_number":795,"balance":31450,"firstname":"Bruce","lastname":"Avila","age":34,"gender":"M","address":"865 Newkirk Placez","employer":"Plasmosis","email":"bruceavila@plasmosis.com","city":"Ada","state":"ID","_is_real":true} +{"index":{"_id":"803"}} +{"account_number":803,"balance":49567,"firstname":"Marissa","lastname":"Spears","age":25,"gender":"M","address":"963 Highland Avenue","employer":"Centregy","email":"marissaspears@centregy.com","city":"Bloomington","state":"MS","_is_real":true} +{"index":{"_id":"808"}} +{"account_number":808,"balance":11251,"firstname":"Nola","lastname":"Quinn","age":20,"gender":"M","address":"863 Wythe Place","employer":"Iplax","email":"nolaquinn@iplax.com","city":"Cuylerville","state":"NH","_is_real":true} +{"index":{"_id":"810"}} +{"account_number":810,"balance":10563,"firstname":"Alyssa","lastname":"Ortega","age":40,"gender":"M","address":"977 Clymer Street","employer":"Eventage","email":"alyssaortega@eventage.com","city":"Convent","state":"SC","_is_real":true} +{"index":{"_id":"815"}} +{"account_number":815,"balance":19336,"firstname":"Guthrie","lastname":"Morse","age":30,"gender":"M","address":"685 Vandalia Avenue","employer":"Gronk","email":"guthriemorse@gronk.com","city":"Fowlerville","state":"OR","_is_real":true} +{"index":{"_id":"822"}} +{"account_number":822,"balance":13024,"firstname":"Hicks","lastname":"Farrell","age":25,"gender":"M","address":"468 Middleton Street","employer":"Zolarex","email":"hicksfarrell@zolarex.com","city":"Columbus","state":"OR","_is_real":true} +{"index":{"_id":"827"}} +{"account_number":827,"balance":37536,"firstname":"Naomi","lastname":"Ball","age":29,"gender":"F","address":"319 Stewart Street","employer":"Isotronic","email":"naomiball@isotronic.com","city":"Trona","state":"NM","_is_real":true} +{"index":{"_id":"834"}} +{"account_number":834,"balance":38049,"firstname":"Sybil","lastname":"Carrillo","age":25,"gender":"M","address":"359 Baughman Place","employer":"Phuel","email":"sybilcarrillo@phuel.com","city":"Kohatk","state":"CT","_is_real":true} +{"index":{"_id":"839"}} +{"account_number":839,"balance":38292,"firstname":"Langley","lastname":"Neal","age":39,"gender":"F","address":"565 Newton Street","employer":"Liquidoc","email":"langleyneal@liquidoc.com","city":"Osage","state":"AL","_is_real":true} +{"index":{"_id":"841"}} +{"account_number":841,"balance":28291,"firstname":"Dalton","lastname":"Waters","age":21,"gender":"M","address":"859 Grand Street","employer":"Malathion","email":"daltonwaters@malathion.com","city":"Tonopah","state":"AZ","_is_real":true} +{"index":{"_id":"846"}} +{"account_number":846,"balance":35099,"firstname":"Maureen","lastname":"Glass","age":22,"gender":"M","address":"140 Amherst Street","employer":"Stelaecor","email":"maureenglass@stelaecor.com","city":"Cucumber","state":"IL","_is_real":true} +{"index":{"_id":"853"}} +{"account_number":853,"balance":38353,"firstname":"Travis","lastname":"Parks","age":40,"gender":"M","address":"930 Bay Avenue","employer":"Pyramax","email":"travisparks@pyramax.com","city":"Gadsden","state":"ND","_is_real":true} +{"index":{"_id":"858"}} +{"account_number":858,"balance":23194,"firstname":"Small","lastname":"Hatfield","age":36,"gender":"M","address":"593 Tennis Court","employer":"Letpro","email":"smallhatfield@letpro.com","city":"Haena","state":"KS","_is_real":true} +{"index":{"_id":"860"}} +{"account_number":860,"balance":23613,"firstname":"Clark","lastname":"Boyd","age":37,"gender":"M","address":"501 Rock Street","employer":"Deepends","email":"clarkboyd@deepends.com","city":"Whitewater","state":"MA","_is_real":true} +{"index":{"_id":"865"}} +{"account_number":865,"balance":10574,"firstname":"Cook","lastname":"Kelley","age":28,"gender":"F","address":"865 Lincoln Terrace","employer":"Quizmo","email":"cookkelley@quizmo.com","city":"Kansas","state":"KY","_is_real":true} +{"index":{"_id":"872"}} +{"account_number":872,"balance":26314,"firstname":"Jane","lastname":"Greer","age":36,"gender":"F","address":"717 Hewes Street","employer":"Newcube","email":"janegreer@newcube.com","city":"Delshire","state":"DE","_is_real":true} +{"index":{"_id":"877"}} +{"account_number":877,"balance":42879,"firstname":"Tracey","lastname":"Ruiz","age":34,"gender":"F","address":"141 Tompkins Avenue","employer":"Waab","email":"traceyruiz@waab.com","city":"Zeba","state":"NM","_is_real":true} +{"index":{"_id":"884"}} +{"account_number":884,"balance":29316,"firstname":"Reva","lastname":"Rosa","age":40,"gender":"M","address":"784 Greene Avenue","employer":"Urbanshee","email":"revarosa@urbanshee.com","city":"Bakersville","state":"MS","_is_real":true} +{"index":{"_id":"889"}} +{"account_number":889,"balance":26464,"firstname":"Fischer","lastname":"Klein","age":38,"gender":"F","address":"948 Juliana Place","employer":"Comtext","email":"fischerklein@comtext.com","city":"Jackpot","state":"PA","_is_real":true} +{"index":{"_id":"891"}} +{"account_number":891,"balance":34829,"firstname":"Jacobson","lastname":"Clemons","age":24,"gender":"F","address":"507 Wilson Street","employer":"Quilm","email":"jacobsonclemons@quilm.com","city":"Muir","state":"TX","_is_real":true} +{"index":{"_id":"896"}} +{"account_number":896,"balance":31947,"firstname":"Buckley","lastname":"Peterson","age":26,"gender":"M","address":"217 Beayer Place","employer":"Earwax","email":"buckleypeterson@earwax.com","city":"Franklin","state":"DE","_is_real":true} +{"index":{"_id":"904"}} +{"account_number":904,"balance":27707,"firstname":"Mendez","lastname":"Mcneil","age":26,"gender":"M","address":"431 Halsey Street","employer":"Macronaut","email":"mendezmcneil@macronaut.com","city":"Troy","state":"OK","_is_real":true} +{"index":{"_id":"909"}} +{"account_number":909,"balance":18421,"firstname":"Stark","lastname":"Lewis","age":36,"gender":"M","address":"409 Tilden Avenue","employer":"Frosnex","email":"starklewis@frosnex.com","city":"Axis","state":"CA","_is_real":true} +{"index":{"_id":"911"}} +{"account_number":911,"balance":42655,"firstname":"Annie","lastname":"Lyons","age":21,"gender":"M","address":"518 Woods Place","employer":"Enerforce","email":"annielyons@enerforce.com","city":"Stagecoach","state":"MA","_is_real":true} +{"index":{"_id":"916"}} +{"account_number":916,"balance":47887,"firstname":"Jarvis","lastname":"Alexander","age":40,"gender":"M","address":"406 Bergen Avenue","employer":"Equitax","email":"jarvisalexander@equitax.com","city":"Haring","state":"KY","_is_real":true} +{"index":{"_id":"923"}} +{"account_number":923,"balance":48466,"firstname":"Mueller","lastname":"Mckee","age":26,"gender":"M","address":"298 Ruby Street","employer":"Luxuria","email":"muellermckee@luxuria.com","city":"Coleville","state":"TN","_is_real":true} +{"index":{"_id":"928"}} +{"account_number":928,"balance":19611,"firstname":"Hester","lastname":"Copeland","age":22,"gender":"F","address":"425 Cropsey Avenue","employer":"Dymi","email":"hestercopeland@dymi.com","city":"Wolcott","state":"NE","_is_real":true} +{"index":{"_id":"930"}} +{"account_number":930,"balance":47257,"firstname":"Kinney","lastname":"Lawson","age":39,"gender":"M","address":"501 Raleigh Place","employer":"Neptide","email":"kinneylawson@neptide.com","city":"Deltaville","state":"MD","_is_real":true} +{"index":{"_id":"935"}} +{"account_number":935,"balance":4959,"firstname":"Flowers","lastname":"Robles","age":30,"gender":"M","address":"201 Hull Street","employer":"Xelegyl","email":"flowersrobles@xelegyl.com","city":"Rehrersburg","state":"AL","_is_real":true} +{"index":{"_id":"942"}} +{"account_number":942,"balance":21299,"firstname":"Hamilton","lastname":"Clayton","age":26,"gender":"M","address":"413 Debevoise Street","employer":"Architax","email":"hamiltonclayton@architax.com","city":"Terlingua","state":"NM","_is_real":true} +{"index":{"_id":"947"}} +{"account_number":947,"balance":22039,"firstname":"Virgie","lastname":"Garza","age":30,"gender":"M","address":"903 Matthews Court","employer":"Plasmox","email":"virgiegarza@plasmox.com","city":"Somerset","state":"WY","_is_real":true} +{"index":{"_id":"954"}} +{"account_number":954,"balance":49404,"firstname":"Jenna","lastname":"Martin","age":22,"gender":"M","address":"688 Hart Street","employer":"Zinca","email":"jennamartin@zinca.com","city":"Oasis","state":"MD","_is_real":true} +{"index":{"_id":"959"}} +{"account_number":959,"balance":34743,"firstname":"Shaffer","lastname":"Cervantes","age":40,"gender":"M","address":"931 Varick Avenue","employer":"Oceanica","email":"shaffercervantes@oceanica.com","city":"Bowie","state":"AL","_is_real":true} +{"index":{"_id":"961"}} +{"account_number":961,"balance":43219,"firstname":"Betsy","lastname":"Hyde","age":27,"gender":"F","address":"183 Junius Street","employer":"Tubalum","email":"betsyhyde@tubalum.com","city":"Driftwood","state":"TX","_is_real":true} +{"index":{"_id":"966"}} +{"account_number":966,"balance":20619,"firstname":"Susanne","lastname":"Rodriguez","age":35,"gender":"F","address":"255 Knickerbocker Avenue","employer":"Comtrek","email":"susannerodriguez@comtrek.com","city":"Trinway","state":"TX","_is_real":true} +{"index":{"_id":"973"}} +{"account_number":973,"balance":45756,"firstname":"Rice","lastname":"Farmer","age":31,"gender":"M","address":"476 Nassau Avenue","employer":"Photobin","email":"ricefarmer@photobin.com","city":"Suitland","state":"ME","_is_real":true} +{"index":{"_id":"978"}} +{"account_number":978,"balance":21459,"firstname":"Melanie","lastname":"Rojas","age":33,"gender":"M","address":"991 Java Street","employer":"Kage","email":"melanierojas@kage.com","city":"Greenock","state":"VT","_is_real":true} +{"index":{"_id":"980"}} +{"account_number":980,"balance":42436,"firstname":"Cash","lastname":"Collier","age":33,"gender":"F","address":"999 Sapphire Street","employer":"Ceprene","email":"cashcollier@ceprene.com","city":"Glidden","state":"AK","_is_real":true} +{"index":{"_id":"985"}} +{"account_number":985,"balance":20083,"firstname":"Martin","lastname":"Gardner","age":28,"gender":"F","address":"644 Fairview Place","employer":"Golistic","email":"martingardner@golistic.com","city":"Connerton","state":"NJ","_is_real":true} +{"index":{"_id":"992"}} +{"account_number":992,"balance":11413,"firstname":"Kristie","lastname":"Kennedy","age":33,"gender":"F","address":"750 Hudson Avenue","employer":"Ludak","email":"kristiekennedy@ludak.com","city":"Warsaw","state":"WY","_is_real":true} +{"index":{"_id":"997"}} +{"account_number":997,"balance":25311,"firstname":"Combs","lastname":"Frederick","age":20,"gender":"M","address":"586 Lloyd Court","employer":"Pathways","email":"combsfrederick@pathways.com","city":"Williamson","state":"CA","_is_real":true} +{"index":{"_id":"3"}} +{"account_number":3,"balance":44947,"firstname":"Levine","lastname":"Burks","age":26,"gender":"F","address":"328 Wilson Avenue","employer":"Amtap","email":"levineburks@amtap.com","city":"Cochranville","state":"HI","_is_real":true} +{"index":{"_id":"8"}} +{"account_number":8,"balance":48868,"firstname":"Jan","lastname":"Burns","age":35,"gender":"M","address":"699 Visitation Place","employer":"Glasstep","email":"janburns@glasstep.com","city":"Wakulla","state":"AZ","_is_real":true} +{"index":{"_id":"10"}} +{"account_number":10,"balance":46170,"firstname":"Dominique","lastname":"Park","age":37,"gender":"F","address":"100 Gatling Place","employer":"Conjurica","email":"dominiquepark@conjurica.com","city":"Omar","state":"NJ","_is_real":true} +{"index":{"_id":"15"}} +{"account_number":15,"balance":43456,"firstname":"Bobbie","lastname":"Sexton","age":21,"gender":"M","address":"232 Sedgwick Place","employer":"Zytrex","email":"bobbiesexton@zytrex.com","city":"Hendersonville","state":"CA","_is_real":true} +{"index":{"_id":"22"}} +{"account_number":22,"balance":40283,"firstname":"Barrera","lastname":"Terrell","age":23,"gender":"F","address":"292 Orange Street","employer":"Steelfab","email":"barreraterrell@steelfab.com","city":"Bynum","state":"ME","_is_real":true} +{"index":{"_id":"27"}} +{"account_number":27,"balance":6176,"firstname":"Meyers","lastname":"Williamson","age":26,"gender":"F","address":"675 Henderson Walk","employer":"Plexia","email":"meyerswilliamson@plexia.com","city":"Richmond","state":"AZ","_is_real":true} +{"index":{"_id":"34"}} +{"account_number":34,"balance":35379,"firstname":"Ellison","lastname":"Kim","age":30,"gender":"F","address":"986 Revere Place","employer":"Signity","email":"ellisonkim@signity.com","city":"Sehili","state":"IL","_is_real":true} +{"index":{"_id":"39"}} +{"account_number":39,"balance":38688,"firstname":"Bowers","lastname":"Mendez","age":22,"gender":"F","address":"665 Bennet Court","employer":"Farmage","email":"bowersmendez@farmage.com","city":"Duryea","state":"PA","_is_real":true} +{"index":{"_id":"41"}} +{"account_number":41,"balance":36060,"firstname":"Hancock","lastname":"Holden","age":20,"gender":"M","address":"625 Gaylord Drive","employer":"Poochies","email":"hancockholden@poochies.com","city":"Alamo","state":"KS","_is_real":true} +{"index":{"_id":"46"}} +{"account_number":46,"balance":12351,"firstname":"Karla","lastname":"Bowman","age":23,"gender":"M","address":"554 Chapel Street","employer":"Undertap","email":"karlabowman@undertap.com","city":"Sylvanite","state":"DC","_is_real":true} +{"index":{"_id":"53"}} +{"account_number":53,"balance":28101,"firstname":"Kathryn","lastname":"Payne","age":29,"gender":"F","address":"467 Louis Place","employer":"Katakana","email":"kathrynpayne@katakana.com","city":"Harviell","state":"SD","_is_real":true} +{"index":{"_id":"58"}} +{"account_number":58,"balance":31697,"firstname":"Marva","lastname":"Cannon","age":40,"gender":"M","address":"993 Highland Place","employer":"Comcubine","email":"marvacannon@comcubine.com","city":"Orviston","state":"MO","_is_real":true} +{"index":{"_id":"60"}} +{"account_number":60,"balance":45955,"firstname":"Maude","lastname":"Casey","age":31,"gender":"F","address":"566 Strauss Street","employer":"Quilch","email":"maudecasey@quilch.com","city":"Enlow","state":"GA","_is_real":true} +{"index":{"_id":"65"}} +{"account_number":65,"balance":23282,"firstname":"Leonor","lastname":"Pruitt","age":24,"gender":"M","address":"974 Terrace Place","employer":"Velos","email":"leonorpruitt@velos.com","city":"Devon","state":"WI","_is_real":true} +{"index":{"_id":"72"}} +{"account_number":72,"balance":9732,"firstname":"Barlow","lastname":"Rhodes","age":25,"gender":"F","address":"891 Clinton Avenue","employer":"Zialactic","email":"barlowrhodes@zialactic.com","city":"Echo","state":"TN","_is_real":true} +{"index":{"_id":"77"}} +{"account_number":77,"balance":5724,"firstname":"Byrd","lastname":"Conley","age":24,"gender":"F","address":"698 Belmont Avenue","employer":"Zidox","email":"byrdconley@zidox.com","city":"Rockbridge","state":"SC","_is_real":true} +{"index":{"_id":"84"}} +{"account_number":84,"balance":3001,"firstname":"Hutchinson","lastname":"Newton","age":34,"gender":"F","address":"553 Locust Street","employer":"Zaggles","email":"hutchinsonnewton@zaggles.com","city":"Snyderville","state":"DC","_is_real":true} +{"index":{"_id":"89"}} +{"account_number":89,"balance":13263,"firstname":"Mcdowell","lastname":"Bradley","age":28,"gender":"M","address":"960 Howard Alley","employer":"Grok","email":"mcdowellbradley@grok.com","city":"Toftrees","state":"TX","_is_real":true} +{"index":{"_id":"91"}} +{"account_number":91,"balance":29799,"firstname":"Vonda","lastname":"Galloway","age":20,"gender":"M","address":"988 Voorhies Avenue","employer":"Illumity","email":"vondagalloway@illumity.com","city":"Holcombe","state":"HI","_is_real":true} +{"index":{"_id":"96"}} +{"account_number":96,"balance":15933,"firstname":"Shirley","lastname":"Edwards","age":38,"gender":"M","address":"817 Caton Avenue","employer":"Equitox","email":"shirleyedwards@equitox.com","city":"Nelson","state":"MA","_is_real":true} +{"index":{"_id":"104"}} +{"account_number":104,"balance":32619,"firstname":"Casey","lastname":"Roth","age":29,"gender":"M","address":"963 Railroad Avenue","employer":"Hotcakes","email":"caseyroth@hotcakes.com","city":"Davenport","state":"OH","_is_real":true} +{"index":{"_id":"109"}} +{"account_number":109,"balance":25812,"firstname":"Gretchen","lastname":"Dawson","age":31,"gender":"M","address":"610 Bethel Loop","employer":"Tetak","email":"gretchendawson@tetak.com","city":"Hailesboro","state":"CO","_is_real":true} +{"index":{"_id":"111"}} +{"account_number":111,"balance":1481,"firstname":"Traci","lastname":"Allison","age":35,"gender":"M","address":"922 Bryant Street","employer":"Enjola","email":"traciallison@enjola.com","city":"Robinette","state":"OR","_is_real":true} +{"index":{"_id":"116"}} +{"account_number":116,"balance":21335,"firstname":"Hobbs","lastname":"Wright","age":24,"gender":"M","address":"965 Temple Court","employer":"Netbook","email":"hobbswright@netbook.com","city":"Strong","state":"CA","_is_real":true} +{"index":{"_id":"123"}} +{"account_number":123,"balance":3079,"firstname":"Cleo","lastname":"Beach","age":27,"gender":"F","address":"653 Haring Street","employer":"Proxsoft","email":"cleobeach@proxsoft.com","city":"Greensburg","state":"ME","_is_real":true} +{"index":{"_id":"128"}} +{"account_number":128,"balance":3556,"firstname":"Mack","lastname":"Bullock","age":34,"gender":"F","address":"462 Ingraham Street","employer":"Terascape","email":"mackbullock@terascape.com","city":"Eureka","state":"PA","_is_real":true} +{"index":{"_id":"130"}} +{"account_number":130,"balance":24171,"firstname":"Roxie","lastname":"Cantu","age":33,"gender":"M","address":"841 Catherine Street","employer":"Skybold","email":"roxiecantu@skybold.com","city":"Deputy","state":"NE","_is_real":true} +{"index":{"_id":"135"}} +{"account_number":135,"balance":24885,"firstname":"Stevenson","lastname":"Crosby","age":40,"gender":"F","address":"473 Boardwalk ","employer":"Accel","email":"stevensoncrosby@accel.com","city":"Norris","state":"OK","_is_real":true} +{"index":{"_id":"142"}} +{"account_number":142,"balance":4544,"firstname":"Vang","lastname":"Hughes","age":27,"gender":"M","address":"357 Landis Court","employer":"Bolax","email":"vanghughes@bolax.com","city":"Emerald","state":"WY","_is_real":true} +{"index":{"_id":"147"}} +{"account_number":147,"balance":35921,"firstname":"Charmaine","lastname":"Whitney","age":28,"gender":"F","address":"484 Seton Place","employer":"Comveyer","email":"charmainewhitney@comveyer.com","city":"Dexter","state":"DC","_is_real":true} +{"index":{"_id":"154"}} +{"account_number":154,"balance":40945,"firstname":"Burns","lastname":"Solis","age":31,"gender":"M","address":"274 Lorraine Street","employer":"Rodemco","email":"burnssolis@rodemco.com","city":"Ballico","state":"WI","_is_real":true} +{"index":{"_id":"159"}} +{"account_number":159,"balance":1696,"firstname":"Alvarez","lastname":"Mack","age":22,"gender":"F","address":"897 Manor Court","employer":"Snorus","email":"alvarezmack@snorus.com","city":"Rosedale","state":"CA","_is_real":true} +{"index":{"_id":"161"}} +{"account_number":161,"balance":4659,"firstname":"Doreen","lastname":"Randall","age":37,"gender":"F","address":"178 Court Street","employer":"Calcula","email":"doreenrandall@calcula.com","city":"Belmont","state":"TX","_is_real":true} +{"index":{"_id":"166"}} +{"account_number":166,"balance":33847,"firstname":"Rutledge","lastname":"Rivas","age":23,"gender":"M","address":"352 Verona Street","employer":"Virxo","email":"rutledgerivas@virxo.com","city":"Brandermill","state":"NE","_is_real":true} +{"index":{"_id":"173"}} +{"account_number":173,"balance":5989,"firstname":"Whitley","lastname":"Blevins","age":32,"gender":"M","address":"127 Brooklyn Avenue","employer":"Pawnagra","email":"whitleyblevins@pawnagra.com","city":"Rodanthe","state":"ND","_is_real":true} +{"index":{"_id":"178"}} +{"account_number":178,"balance":36735,"firstname":"Clements","lastname":"Finley","age":39,"gender":"F","address":"270 Story Court","employer":"Imaginart","email":"clementsfinley@imaginart.com","city":"Lookingglass","state":"MN","_is_real":true} +{"index":{"_id":"180"}} +{"account_number":180,"balance":34236,"firstname":"Ursula","lastname":"Goodman","age":32,"gender":"F","address":"414 Clinton Street","employer":"Earthmark","email":"ursulagoodman@earthmark.com","city":"Rote","state":"AR","_is_real":true} +{"index":{"_id":"185"}} +{"account_number":185,"balance":43532,"firstname":"Laurel","lastname":"Cline","age":40,"gender":"M","address":"788 Fenimore Street","employer":"Prismatic","email":"laurelcline@prismatic.com","city":"Frank","state":"UT","_is_real":true} +{"index":{"_id":"192"}} +{"account_number":192,"balance":23508,"firstname":"Ramsey","lastname":"Carr","age":31,"gender":"F","address":"209 Williamsburg Street","employer":"Strezzo","email":"ramseycarr@strezzo.com","city":"Grapeview","state":"NM","_is_real":true} +{"index":{"_id":"197"}} +{"account_number":197,"balance":17246,"firstname":"Sweet","lastname":"Sanders","age":33,"gender":"F","address":"712 Homecrest Court","employer":"Isosure","email":"sweetsanders@isosure.com","city":"Sheatown","state":"VT","_is_real":true} +{"index":{"_id":"200"}} +{"account_number":200,"balance":26210,"firstname":"Teri","lastname":"Hester","age":39,"gender":"M","address":"653 Abbey Court","employer":"Electonic","email":"terihester@electonic.com","city":"Martell","state":"MD","_is_real":true} +{"index":{"_id":"205"}} +{"account_number":205,"balance":45493,"firstname":"Johnson","lastname":"Chang","age":28,"gender":"F","address":"331 John Street","employer":"Gleamink","email":"johnsonchang@gleamink.com","city":"Sultana","state":"KS","_is_real":true} +{"index":{"_id":"212"}} +{"account_number":212,"balance":10299,"firstname":"Marisol","lastname":"Fischer","age":39,"gender":"M","address":"362 Prince Street","employer":"Autograte","email":"marisolfischer@autograte.com","city":"Oley","state":"SC","_is_real":true} +{"index":{"_id":"217"}} +{"account_number":217,"balance":33730,"firstname":"Sally","lastname":"Mccoy","age":38,"gender":"F","address":"854 Corbin Place","employer":"Omnigog","email":"sallymccoy@omnigog.com","city":"Escondida","state":"FL","_is_real":true} +{"index":{"_id":"224"}} +{"account_number":224,"balance":42708,"firstname":"Billie","lastname":"Nixon","age":28,"gender":"F","address":"241 Kaufman Place","employer":"Xanide","email":"billienixon@xanide.com","city":"Chapin","state":"NY","_is_real":true} +{"index":{"_id":"229"}} +{"account_number":229,"balance":2740,"firstname":"Jana","lastname":"Hensley","age":30,"gender":"M","address":"176 Erasmus Street","employer":"Isotrack","email":"janahensley@isotrack.com","city":"Caledonia","state":"ME","_is_real":true} +{"index":{"_id":"231"}} +{"account_number":231,"balance":46180,"firstname":"Essie","lastname":"Clarke","age":34,"gender":"F","address":"308 Harbor Lane","employer":"Pharmacon","email":"essieclarke@pharmacon.com","city":"Fillmore","state":"MS","_is_real":true} +{"index":{"_id":"236"}} +{"account_number":236,"balance":41200,"firstname":"Suzanne","lastname":"Bird","age":39,"gender":"F","address":"219 Luquer Street","employer":"Imant","email":"suzannebird@imant.com","city":"Bainbridge","state":"NY","_is_real":true} +{"index":{"_id":"243"}} +{"account_number":243,"balance":29902,"firstname":"Evangelina","lastname":"Perez","age":20,"gender":"M","address":"787 Joval Court","employer":"Keengen","email":"evangelinaperez@keengen.com","city":"Mulberry","state":"SD","_is_real":true} +{"index":{"_id":"248"}} +{"account_number":248,"balance":49989,"firstname":"West","lastname":"England","age":36,"gender":"M","address":"717 Hendrickson Place","employer":"Obliq","email":"westengland@obliq.com","city":"Maury","state":"WA","_is_real":true} +{"index":{"_id":"250"}} +{"account_number":250,"balance":27893,"firstname":"Earlene","lastname":"Ellis","age":39,"gender":"F","address":"512 Bay Street","employer":"Codact","email":"earleneellis@codact.com","city":"Sunwest","state":"GA","_is_real":true} +{"index":{"_id":"255"}} +{"account_number":255,"balance":49339,"firstname":"Iva","lastname":"Rivers","age":38,"gender":"M","address":"470 Rost Place","employer":"Mantrix","email":"ivarivers@mantrix.com","city":"Disautel","state":"MD","_is_real":true} +{"index":{"_id":"262"}} +{"account_number":262,"balance":30289,"firstname":"Tameka","lastname":"Levine","age":36,"gender":"F","address":"815 Atlantic Avenue","employer":"Acium","email":"tamekalevine@acium.com","city":"Winchester","state":"SD","_is_real":true} +{"index":{"_id":"267"}} +{"account_number":267,"balance":42753,"firstname":"Weeks","lastname":"Castillo","age":21,"gender":"F","address":"526 Holt Court","employer":"Talendula","email":"weekscastillo@talendula.com","city":"Washington","state":"NV","_is_real":true} +{"index":{"_id":"274"}} +{"account_number":274,"balance":12104,"firstname":"Frieda","lastname":"House","age":33,"gender":"F","address":"171 Banker Street","employer":"Quonk","email":"friedahouse@quonk.com","city":"Aberdeen","state":"NJ","_is_real":true} +{"index":{"_id":"279"}} +{"account_number":279,"balance":15904,"firstname":"Chapman","lastname":"Hart","age":32,"gender":"F","address":"902 Bliss Terrace","employer":"Kongene","email":"chapmanhart@kongene.com","city":"Bradenville","state":"NJ","_is_real":true} +{"index":{"_id":"281"}} +{"account_number":281,"balance":39830,"firstname":"Bean","lastname":"Aguirre","age":20,"gender":"F","address":"133 Pilling Street","employer":"Amril","email":"beanaguirre@amril.com","city":"Waterview","state":"TX","_is_real":true} +{"index":{"_id":"286"}} +{"account_number":286,"balance":39063,"firstname":"Rosetta","lastname":"Turner","age":35,"gender":"M","address":"169 Jefferson Avenue","employer":"Spacewax","email":"rosettaturner@spacewax.com","city":"Stewart","state":"MO","_is_real":true} +{"index":{"_id":"293"}} +{"account_number":293,"balance":29867,"firstname":"Cruz","lastname":"Carver","age":28,"gender":"F","address":"465 Boerum Place","employer":"Vitricomp","email":"cruzcarver@vitricomp.com","city":"Crayne","state":"CO","_is_real":true} +{"index":{"_id":"298"}} +{"account_number":298,"balance":34334,"firstname":"Bullock","lastname":"Marsh","age":20,"gender":"M","address":"589 Virginia Place","employer":"Renovize","email":"bullockmarsh@renovize.com","city":"Coinjock","state":"UT","_is_real":true} +{"index":{"_id":"301"}} +{"account_number":301,"balance":16782,"firstname":"Minerva","lastname":"Graham","age":35,"gender":"M","address":"532 Harrison Place","employer":"Sureplex","email":"minervagraham@sureplex.com","city":"Belleview","state":"GA","_is_real":true} +{"index":{"_id":"306"}} +{"account_number":306,"balance":2171,"firstname":"Hensley","lastname":"Hardin","age":40,"gender":"M","address":"196 Maujer Street","employer":"Neocent","email":"hensleyhardin@neocent.com","city":"Reinerton","state":"HI","_is_real":true} +{"index":{"_id":"313"}} +{"account_number":313,"balance":34108,"firstname":"Alston","lastname":"Henderson","age":36,"gender":"F","address":"132 Prescott Place","employer":"Prosure","email":"alstonhenderson@prosure.com","city":"Worton","state":"IA","_is_real":true} +{"index":{"_id":"318"}} +{"account_number":318,"balance":8512,"firstname":"Nichole","lastname":"Pearson","age":34,"gender":"F","address":"656 Lacon Court","employer":"Yurture","email":"nicholepearson@yurture.com","city":"Juarez","state":"MO","_is_real":true} +{"index":{"_id":"320"}} +{"account_number":320,"balance":34521,"firstname":"Patti","lastname":"Brennan","age":37,"gender":"F","address":"870 Degraw Street","employer":"Cognicode","email":"pattibrennan@cognicode.com","city":"Torboy","state":"FL","_is_real":true} +{"index":{"_id":"325"}} +{"account_number":325,"balance":1956,"firstname":"Magdalena","lastname":"Simmons","age":25,"gender":"F","address":"681 Townsend Street","employer":"Geekosis","email":"magdalenasimmons@geekosis.com","city":"Sterling","state":"CA","_is_real":true} +{"index":{"_id":"332"}} +{"account_number":332,"balance":37770,"firstname":"Shepherd","lastname":"Davenport","age":28,"gender":"F","address":"586 Montague Terrace","employer":"Ecraze","email":"shepherddavenport@ecraze.com","city":"Accoville","state":"NM","_is_real":true} +{"index":{"_id":"337"}} +{"account_number":337,"balance":43432,"firstname":"Monroe","lastname":"Stafford","age":37,"gender":"F","address":"183 Seigel Street","employer":"Centuria","email":"monroestafford@centuria.com","city":"Camino","state":"DE","_is_real":true} +{"index":{"_id":"344"}} +{"account_number":344,"balance":42654,"firstname":"Sasha","lastname":"Baxter","age":35,"gender":"F","address":"700 Bedford Place","employer":"Callflex","email":"sashabaxter@callflex.com","city":"Campo","state":"MI","_is_real":true} +{"index":{"_id":"349"}} +{"account_number":349,"balance":24180,"firstname":"Allison","lastname":"Fitzpatrick","age":22,"gender":"F","address":"913 Arlington Avenue","employer":"Veraq","email":"allisonfitzpatrick@veraq.com","city":"Marbury","state":"TX","_is_real":true} +{"index":{"_id":"351"}} +{"account_number":351,"balance":47089,"firstname":"Hendrix","lastname":"Stephens","age":29,"gender":"M","address":"181 Beaver Street","employer":"Recrisys","email":"hendrixstephens@recrisys.com","city":"Denio","state":"OR","_is_real":true} +{"index":{"_id":"356"}} +{"account_number":356,"balance":34540,"firstname":"Lourdes","lastname":"Valdez","age":20,"gender":"F","address":"700 Anchorage Place","employer":"Interloo","email":"lourdesvaldez@interloo.com","city":"Goldfield","state":"OK","_is_real":true} +{"index":{"_id":"363"}} +{"account_number":363,"balance":34007,"firstname":"Peggy","lastname":"Bright","age":21,"gender":"M","address":"613 Engert Avenue","employer":"Inventure","email":"peggybright@inventure.com","city":"Chautauqua","state":"ME","_is_real":true} +{"index":{"_id":"368"}} +{"account_number":368,"balance":23535,"firstname":"Hooper","lastname":"Tyson","age":39,"gender":"M","address":"892 Taaffe Place","employer":"Zaggle","email":"hoopertyson@zaggle.com","city":"Nutrioso","state":"ME","_is_real":true} +{"index":{"_id":"370"}} +{"account_number":370,"balance":28499,"firstname":"Oneill","lastname":"Carney","age":25,"gender":"F","address":"773 Adelphi Street","employer":"Bedder","email":"oneillcarney@bedder.com","city":"Yorklyn","state":"FL","_is_real":true} +{"index":{"_id":"375"}} +{"account_number":375,"balance":23860,"firstname":"Phoebe","lastname":"Patton","age":25,"gender":"M","address":"564 Hale Avenue","employer":"Xoggle","email":"phoebepatton@xoggle.com","city":"Brule","state":"NM","_is_real":true} +{"index":{"_id":"382"}} +{"account_number":382,"balance":42061,"firstname":"Finley","lastname":"Singleton","age":37,"gender":"F","address":"407 Clay Street","employer":"Quarex","email":"finleysingleton@quarex.com","city":"Bedias","state":"LA","_is_real":true} +{"index":{"_id":"387"}} +{"account_number":387,"balance":35916,"firstname":"April","lastname":"Hill","age":29,"gender":"M","address":"818 Bayard Street","employer":"Kengen","email":"aprilhill@kengen.com","city":"Chloride","state":"NC","_is_real":true} +{"index":{"_id":"394"}} +{"account_number":394,"balance":6121,"firstname":"Lorrie","lastname":"Nunez","age":38,"gender":"M","address":"221 Ralph Avenue","employer":"Bullzone","email":"lorrienunez@bullzone.com","city":"Longoria","state":"ID","_is_real":true} +{"index":{"_id":"399"}} +{"account_number":399,"balance":32587,"firstname":"Carmela","lastname":"Franks","age":23,"gender":"M","address":"617 Dewey Place","employer":"Zensure","email":"carmelafranks@zensure.com","city":"Sanders","state":"DC","_is_real":true} +{"index":{"_id":"402"}} +{"account_number":402,"balance":1282,"firstname":"Pacheco","lastname":"Rosales","age":32,"gender":"M","address":"538 Pershing Loop","employer":"Circum","email":"pachecorosales@circum.com","city":"Elbert","state":"ID","_is_real":true} +{"index":{"_id":"407"}} +{"account_number":407,"balance":36417,"firstname":"Gilda","lastname":"Jacobson","age":29,"gender":"F","address":"883 Loring Avenue","employer":"Comveyor","email":"gildajacobson@comveyor.com","city":"Topaz","state":"NH","_is_real":true} +{"index":{"_id":"414"}} +{"account_number":414,"balance":17506,"firstname":"Conway","lastname":"Daugherty","age":37,"gender":"F","address":"643 Kermit Place","employer":"Lyria","email":"conwaydaugherty@lyria.com","city":"Vaughn","state":"NV","_is_real":true} +{"index":{"_id":"419"}} +{"account_number":419,"balance":34847,"firstname":"Helen","lastname":"Montoya","age":29,"gender":"F","address":"736 Kingsland Avenue","employer":"Hairport","email":"helenmontoya@hairport.com","city":"Edinburg","state":"NE","_is_real":true} +{"index":{"_id":"421"}} +{"account_number":421,"balance":46868,"firstname":"Tamika","lastname":"Mccall","age":27,"gender":"F","address":"764 Bragg Court","employer":"Eventix","email":"tamikamccall@eventix.com","city":"Tivoli","state":"RI","_is_real":true} +{"index":{"_id":"426"}} +{"account_number":426,"balance":4499,"firstname":"Julie","lastname":"Parsons","age":31,"gender":"M","address":"768 Keap Street","employer":"Goko","email":"julieparsons@goko.com","city":"Coldiron","state":"VA","_is_real":true} +{"index":{"_id":"433"}} +{"account_number":433,"balance":19266,"firstname":"Wilkinson","lastname":"Flowers","age":39,"gender":"M","address":"154 Douglass Street","employer":"Xsports","email":"wilkinsonflowers@xsports.com","city":"Coultervillle","state":"MN","_is_real":true} +{"index":{"_id":"438"}} +{"account_number":438,"balance":16367,"firstname":"Walter","lastname":"Velez","age":27,"gender":"F","address":"931 Farragut Road","employer":"Virva","email":"waltervelez@virva.com","city":"Tyro","state":"WV","_is_real":true} +{"index":{"_id":"440"}} +{"account_number":440,"balance":41590,"firstname":"Ray","lastname":"Wiley","age":31,"gender":"F","address":"102 Barwell Terrace","employer":"Polaria","email":"raywiley@polaria.com","city":"Hardyville","state":"IA","_is_real":true} +{"index":{"_id":"445"}} +{"account_number":445,"balance":41178,"firstname":"Rodriguez","lastname":"Macias","age":34,"gender":"M","address":"164 Boerum Street","employer":"Xylar","email":"rodriguezmacias@xylar.com","city":"Riner","state":"AL","_is_real":true} +{"index":{"_id":"452"}} +{"account_number":452,"balance":3589,"firstname":"Blackwell","lastname":"Delaney","age":39,"gender":"F","address":"443 Sackett Street","employer":"Imkan","email":"blackwelldelaney@imkan.com","city":"Gasquet","state":"DC","_is_real":true} +{"index":{"_id":"457"}} +{"account_number":457,"balance":14057,"firstname":"Bush","lastname":"Gordon","age":34,"gender":"M","address":"975 Dakota Place","employer":"Softmicro","email":"bushgordon@softmicro.com","city":"Chemung","state":"PA","_is_real":true} +{"index":{"_id":"464"}} +{"account_number":464,"balance":20504,"firstname":"Cobb","lastname":"Humphrey","age":21,"gender":"M","address":"823 Sunnyside Avenue","employer":"Apexia","email":"cobbhumphrey@apexia.com","city":"Wintersburg","state":"NY","_is_real":true} +{"index":{"_id":"469"}} +{"account_number":469,"balance":26509,"firstname":"Marci","lastname":"Shepherd","age":26,"gender":"M","address":"565 Hall Street","employer":"Shadease","email":"marcishepherd@shadease.com","city":"Springhill","state":"IL","_is_real":true} +{"index":{"_id":"471"}} +{"account_number":471,"balance":7629,"firstname":"Juana","lastname":"Silva","age":36,"gender":"M","address":"249 Amity Street","employer":"Artworlds","email":"juanasilva@artworlds.com","city":"Norfolk","state":"TX","_is_real":true} +{"index":{"_id":"476"}} +{"account_number":476,"balance":33386,"firstname":"Silva","lastname":"Marks","age":31,"gender":"F","address":"183 Eldert Street","employer":"Medifax","email":"silvamarks@medifax.com","city":"Hachita","state":"RI","_is_real":true} +{"index":{"_id":"483"}} +{"account_number":483,"balance":6344,"firstname":"Kelley","lastname":"Harper","age":29,"gender":"M","address":"758 Preston Court","employer":"Xyqag","email":"kelleyharper@xyqag.com","city":"Healy","state":"IA","_is_real":true} +{"index":{"_id":"488"}} +{"account_number":488,"balance":6289,"firstname":"Wilma","lastname":"Hopkins","age":38,"gender":"M","address":"428 Lee Avenue","employer":"Entality","email":"wilmahopkins@entality.com","city":"Englevale","state":"WI","_is_real":true} +{"index":{"_id":"490"}} +{"account_number":490,"balance":1447,"firstname":"Strong","lastname":"Hendrix","age":26,"gender":"F","address":"134 Beach Place","employer":"Duoflex","email":"stronghendrix@duoflex.com","city":"Allentown","state":"ND","_is_real":true} +{"index":{"_id":"495"}} +{"account_number":495,"balance":13478,"firstname":"Abigail","lastname":"Nichols","age":40,"gender":"F","address":"887 President Street","employer":"Enquility","email":"abigailnichols@enquility.com","city":"Bagtown","state":"NM","_is_real":true} +{"index":{"_id":"503"}} +{"account_number":503,"balance":42649,"firstname":"Leta","lastname":"Stout","age":39,"gender":"F","address":"518 Bowery Street","employer":"Pivitol","email":"letastout@pivitol.com","city":"Boonville","state":"ND","_is_real":true} +{"index":{"_id":"508"}} +{"account_number":508,"balance":41300,"firstname":"Lawrence","lastname":"Mathews","age":27,"gender":"F","address":"987 Rose Street","employer":"Deviltoe","email":"lawrencemathews@deviltoe.com","city":"Woodburn","state":"FL","_is_real":true} +{"index":{"_id":"510"}} +{"account_number":510,"balance":48504,"firstname":"Petty","lastname":"Sykes","age":28,"gender":"M","address":"566 Village Road","employer":"Nebulean","email":"pettysykes@nebulean.com","city":"Wedgewood","state":"MO","_is_real":true} +{"index":{"_id":"515"}} +{"account_number":515,"balance":18531,"firstname":"Lott","lastname":"Keller","age":27,"gender":"M","address":"827 Miami Court","employer":"Translink","email":"lottkeller@translink.com","city":"Gila","state":"TX","_is_real":true} +{"index":{"_id":"522"}} +{"account_number":522,"balance":19879,"firstname":"Faulkner","lastname":"Garrett","age":29,"gender":"F","address":"396 Grove Place","employer":"Pigzart","email":"faulknergarrett@pigzart.com","city":"Felt","state":"AR","_is_real":true} +{"index":{"_id":"527"}} +{"account_number":527,"balance":2028,"firstname":"Carver","lastname":"Peters","age":35,"gender":"M","address":"816 Victor Road","employer":"Housedown","email":"carverpeters@housedown.com","city":"Nadine","state":"MD","_is_real":true} +{"index":{"_id":"534"}} +{"account_number":534,"balance":20470,"firstname":"Cristina","lastname":"Russo","age":25,"gender":"F","address":"500 Highlawn Avenue","employer":"Cyclonica","email":"cristinarusso@cyclonica.com","city":"Gorst","state":"KS","_is_real":true} +{"index":{"_id":"539"}} +{"account_number":539,"balance":24560,"firstname":"Tami","lastname":"Maddox","age":23,"gender":"F","address":"741 Pineapple Street","employer":"Accidency","email":"tamimaddox@accidency.com","city":"Kennedyville","state":"OH","_is_real":true} +{"index":{"_id":"541"}} +{"account_number":541,"balance":42915,"firstname":"Logan","lastname":"Burke","age":32,"gender":"M","address":"904 Clarendon Road","employer":"Overplex","email":"loganburke@overplex.com","city":"Johnsonburg","state":"OH","_is_real":true} +{"index":{"_id":"546"}} +{"account_number":546,"balance":43242,"firstname":"Bernice","lastname":"Sims","age":33,"gender":"M","address":"382 Columbia Street","employer":"Verbus","email":"bernicesims@verbus.com","city":"Sena","state":"KY","_is_real":true} +{"index":{"_id":"553"}} +{"account_number":553,"balance":28390,"firstname":"Aimee","lastname":"Cohen","age":28,"gender":"M","address":"396 Lafayette Avenue","employer":"Eplode","email":"aimeecohen@eplode.com","city":"Thatcher","state":"NJ","_is_real":true} +{"index":{"_id":"558"}} +{"account_number":558,"balance":8922,"firstname":"Horne","lastname":"Valenzuela","age":20,"gender":"F","address":"979 Kensington Street","employer":"Isoternia","email":"hornevalenzuela@isoternia.com","city":"Greenbush","state":"NC","_is_real":true} +{"index":{"_id":"560"}} +{"account_number":560,"balance":24514,"firstname":"Felecia","lastname":"Oneill","age":26,"gender":"M","address":"995 Autumn Avenue","employer":"Mediot","email":"feleciaoneill@mediot.com","city":"Joppa","state":"IN","_is_real":true} +{"index":{"_id":"565"}} +{"account_number":565,"balance":15197,"firstname":"Taylor","lastname":"Ingram","age":37,"gender":"F","address":"113 Will Place","employer":"Lyrichord","email":"tayloringram@lyrichord.com","city":"Collins","state":"ME","_is_real":true} +{"index":{"_id":"572"}} +{"account_number":572,"balance":49355,"firstname":"Therese","lastname":"Espinoza","age":20,"gender":"M","address":"994 Chester Court","employer":"Gonkle","email":"thereseespinoza@gonkle.com","city":"Hayes","state":"UT","_is_real":true} +{"index":{"_id":"577"}} +{"account_number":577,"balance":21398,"firstname":"Gilbert","lastname":"Serrano","age":38,"gender":"F","address":"294 Troutman Street","employer":"Senmao","email":"gilbertserrano@senmao.com","city":"Greer","state":"MT","_is_real":true} +{"index":{"_id":"584"}} +{"account_number":584,"balance":5346,"firstname":"Pearson","lastname":"Bryant","age":40,"gender":"F","address":"971 Heyward Street","employer":"Anacho","email":"pearsonbryant@anacho.com","city":"Bluffview","state":"MN","_is_real":true} +{"index":{"_id":"589"}} +{"account_number":589,"balance":33260,"firstname":"Ericka","lastname":"Cote","age":39,"gender":"F","address":"425 Bath Avenue","employer":"Venoflex","email":"erickacote@venoflex.com","city":"Blue","state":"CT","_is_real":true} +{"index":{"_id":"591"}} +{"account_number":591,"balance":48997,"firstname":"Rivers","lastname":"Macdonald","age":34,"gender":"F","address":"919 Johnson Street","employer":"Ziore","email":"riversmacdonald@ziore.com","city":"Townsend","state":"IL","_is_real":true} +{"index":{"_id":"596"}} +{"account_number":596,"balance":4063,"firstname":"Letitia","lastname":"Walker","age":26,"gender":"F","address":"963 Vanderveer Place","employer":"Zizzle","email":"letitiawalker@zizzle.com","city":"Rossmore","state":"ID","_is_real":true} +{"index":{"_id":"604"}} +{"account_number":604,"balance":10675,"firstname":"Isabel","lastname":"Gilliam","age":23,"gender":"M","address":"854 Broadway ","employer":"Zenthall","email":"isabelgilliam@zenthall.com","city":"Ventress","state":"WI","_is_real":true} +{"index":{"_id":"609"}} +{"account_number":609,"balance":28586,"firstname":"Montgomery","lastname":"Washington","age":30,"gender":"M","address":"169 Schroeders Avenue","employer":"Kongle","email":"montgomerywashington@kongle.com","city":"Croom","state":"AZ","_is_real":true} +{"index":{"_id":"611"}} +{"account_number":611,"balance":17528,"firstname":"Katherine","lastname":"Prince","age":33,"gender":"F","address":"705 Elm Avenue","employer":"Zillacon","email":"katherineprince@zillacon.com","city":"Rew","state":"MI","_is_real":true} +{"index":{"_id":"616"}} +{"account_number":616,"balance":25276,"firstname":"Jessie","lastname":"Mayer","age":35,"gender":"F","address":"683 Chester Avenue","employer":"Emtrak","email":"jessiemayer@emtrak.com","city":"Marysville","state":"HI","_is_real":true} +{"index":{"_id":"623"}} +{"account_number":623,"balance":20514,"firstname":"Rose","lastname":"Combs","age":32,"gender":"F","address":"312 Grimes Road","employer":"Aquamate","email":"rosecombs@aquamate.com","city":"Fostoria","state":"OH","_is_real":true} +{"index":{"_id":"628"}} +{"account_number":628,"balance":42736,"firstname":"Buckner","lastname":"Chen","age":37,"gender":"M","address":"863 Rugby Road","employer":"Jamnation","email":"bucknerchen@jamnation.com","city":"Camas","state":"TX","_is_real":true} +{"index":{"_id":"630"}} +{"account_number":630,"balance":46060,"firstname":"Leanne","lastname":"Jones","age":31,"gender":"M","address":"451 Bayview Avenue","employer":"Wazzu","email":"leannejones@wazzu.com","city":"Kylertown","state":"OK","_is_real":true} +{"index":{"_id":"635"}} +{"account_number":635,"balance":44705,"firstname":"Norman","lastname":"Gilmore","age":33,"gender":"M","address":"330 Gates Avenue","employer":"Comfirm","email":"normangilmore@comfirm.com","city":"Riceville","state":"TN","_is_real":true} +{"index":{"_id":"642"}} +{"account_number":642,"balance":32852,"firstname":"Reyna","lastname":"Harris","age":35,"gender":"M","address":"305 Powell Street","employer":"Bedlam","email":"reynaharris@bedlam.com","city":"Florence","state":"KS","_is_real":true} +{"index":{"_id":"647"}} +{"account_number":647,"balance":10147,"firstname":"Annabelle","lastname":"Velazquez","age":30,"gender":"M","address":"299 Kensington Walk","employer":"Sealoud","email":"annabellevelazquez@sealoud.com","city":"Soudan","state":"ME","_is_real":true} +{"index":{"_id":"654"}} +{"account_number":654,"balance":38695,"firstname":"Armstrong","lastname":"Frazier","age":25,"gender":"M","address":"899 Seeley Street","employer":"Zensor","email":"armstrongfrazier@zensor.com","city":"Cherokee","state":"UT","_is_real":true} +{"index":{"_id":"659"}} +{"account_number":659,"balance":29648,"firstname":"Dorsey","lastname":"Sosa","age":40,"gender":"M","address":"270 Aberdeen Street","employer":"Daycore","email":"dorseysosa@daycore.com","city":"Chamberino","state":"SC","_is_real":true} +{"index":{"_id":"661"}} +{"account_number":661,"balance":3679,"firstname":"Joanne","lastname":"Spencer","age":39,"gender":"F","address":"910 Montauk Avenue","employer":"Visalia","email":"joannespencer@visalia.com","city":"Valmy","state":"NH","_is_real":true} +{"index":{"_id":"666"}} +{"account_number":666,"balance":13880,"firstname":"Mcguire","lastname":"Lloyd","age":40,"gender":"F","address":"658 Just Court","employer":"Centrexin","email":"mcguirelloyd@centrexin.com","city":"Warren","state":"MT","_is_real":true} +{"index":{"_id":"673"}} +{"account_number":673,"balance":11303,"firstname":"Mcdaniel","lastname":"Harrell","age":33,"gender":"M","address":"565 Montgomery Place","employer":"Eyeris","email":"mcdanielharrell@eyeris.com","city":"Garnet","state":"NV","_is_real":true} +{"index":{"_id":"678"}} +{"account_number":678,"balance":43663,"firstname":"Ruby","lastname":"Shaffer","age":28,"gender":"M","address":"350 Clark Street","employer":"Comtrail","email":"rubyshaffer@comtrail.com","city":"Aurora","state":"MA","_is_real":true} +{"index":{"_id":"680"}} +{"account_number":680,"balance":31561,"firstname":"Melton","lastname":"Camacho","age":32,"gender":"F","address":"771 Montana Place","employer":"Insuresys","email":"meltoncamacho@insuresys.com","city":"Sparkill","state":"IN","_is_real":true} +{"index":{"_id":"685"}} +{"account_number":685,"balance":22249,"firstname":"Yesenia","lastname":"Rowland","age":24,"gender":"F","address":"193 Dekalb Avenue","employer":"Coriander","email":"yeseniarowland@coriander.com","city":"Lupton","state":"NC","_is_real":true} +{"index":{"_id":"692"}} +{"account_number":692,"balance":10435,"firstname":"Haney","lastname":"Barlow","age":21,"gender":"F","address":"267 Lenox Road","employer":"Egypto","email":"haneybarlow@egypto.com","city":"Detroit","state":"IN","_is_real":true} +{"index":{"_id":"697"}} +{"account_number":697,"balance":48745,"firstname":"Mallory","lastname":"Emerson","age":24,"gender":"F","address":"318 Dunne Court","employer":"Exoplode","email":"malloryemerson@exoplode.com","city":"Montura","state":"LA","_is_real":true} +{"index":{"_id":"700"}} +{"account_number":700,"balance":19164,"firstname":"Patel","lastname":"Durham","age":21,"gender":"F","address":"440 King Street","employer":"Icology","email":"pateldurham@icology.com","city":"Mammoth","state":"IL","_is_real":true} +{"index":{"_id":"705"}} +{"account_number":705,"balance":28415,"firstname":"Krystal","lastname":"Cross","age":22,"gender":"M","address":"604 Drew Street","employer":"Tubesys","email":"krystalcross@tubesys.com","city":"Dalton","state":"MO","_is_real":true} +{"index":{"_id":"712"}} +{"account_number":712,"balance":12459,"firstname":"Butler","lastname":"Alston","age":37,"gender":"M","address":"486 Hemlock Street","employer":"Quordate","email":"butleralston@quordate.com","city":"Verdi","state":"MS","_is_real":true} +{"index":{"_id":"717"}} +{"account_number":717,"balance":29270,"firstname":"Erickson","lastname":"Mcdonald","age":31,"gender":"M","address":"873 Franklin Street","employer":"Exotechno","email":"ericksonmcdonald@exotechno.com","city":"Jessie","state":"MS","_is_real":true} +{"index":{"_id":"724"}} +{"account_number":724,"balance":12548,"firstname":"Hopper","lastname":"Peck","age":31,"gender":"M","address":"849 Hendrickson Street","employer":"Uxmox","email":"hopperpeck@uxmox.com","city":"Faxon","state":"UT","_is_real":true} +{"index":{"_id":"729"}} +{"account_number":729,"balance":41812,"firstname":"Katy","lastname":"Rivera","age":36,"gender":"F","address":"791 Olive Street","employer":"Blurrybus","email":"katyrivera@blurrybus.com","city":"Innsbrook","state":"MI","_is_real":true} +{"index":{"_id":"731"}} +{"account_number":731,"balance":4994,"firstname":"Lorene","lastname":"Weiss","age":35,"gender":"M","address":"990 Ocean Court","employer":"Comvoy","email":"loreneweiss@comvoy.com","city":"Lavalette","state":"WI","_is_real":true} +{"index":{"_id":"736"}} +{"account_number":736,"balance":28677,"firstname":"Rogers","lastname":"Mcmahon","age":21,"gender":"F","address":"423 Cameron Court","employer":"Brainclip","email":"rogersmcmahon@brainclip.com","city":"Saddlebrooke","state":"FL","_is_real":true} +{"index":{"_id":"743"}} +{"account_number":743,"balance":14077,"firstname":"Susana","lastname":"Moody","age":23,"gender":"M","address":"842 Fountain Avenue","employer":"Bitrex","email":"susanamoody@bitrex.com","city":"Temperanceville","state":"TN","_is_real":true} +{"index":{"_id":"748"}} +{"account_number":748,"balance":38060,"firstname":"Ford","lastname":"Branch","age":25,"gender":"M","address":"926 Cypress Avenue","employer":"Buzzness","email":"fordbranch@buzzness.com","city":"Beason","state":"DC","_is_real":true} +{"index":{"_id":"750"}} +{"account_number":750,"balance":40481,"firstname":"Cherie","lastname":"Brooks","age":20,"gender":"F","address":"601 Woodhull Street","employer":"Kaggle","email":"cheriebrooks@kaggle.com","city":"Groton","state":"MA","_is_real":true} +{"index":{"_id":"755"}} +{"account_number":755,"balance":43878,"firstname":"Bartlett","lastname":"Conway","age":22,"gender":"M","address":"453 Times Placez","employer":"Konnect","email":"bartlettconway@konnect.com","city":"Belva","state":"VT","_is_real":true} +{"index":{"_id":"762"}} +{"account_number":762,"balance":10291,"firstname":"Amanda","lastname":"Head","age":20,"gender":"F","address":"990 Ocean Parkway","employer":"Zentury","email":"amandahead@zentury.com","city":"Hegins","state":"AR","_is_real":true} +{"index":{"_id":"767"}} +{"account_number":767,"balance":26220,"firstname":"Anthony","lastname":"Sutton","age":27,"gender":"F","address":"179 Fayette Street","employer":"Xiix","email":"anthonysutton@xiix.com","city":"Iberia","state":"TN","_is_real":true} +{"index":{"_id":"774"}} +{"account_number":774,"balance":35287,"firstname":"Lynnette","lastname":"Alvarez","age":38,"gender":"F","address":"991 Brightwater Avenue","employer":"Gink","email":"lynnettealvarez@gink.com","city":"Leola","state":"NC","_is_real":true} +{"index":{"_id":"779"}} +{"account_number":779,"balance":40983,"firstname":"Maggie","lastname":"Pace","age":32,"gender":"F","address":"104 Harbor Court","employer":"Bulljuice","email":"maggiepace@bulljuice.com","city":"Floris","state":"MA","_is_real":true} +{"index":{"_id":"781"}} +{"account_number":781,"balance":29961,"firstname":"Sanford","lastname":"Mullen","age":26,"gender":"F","address":"879 Dover Street","employer":"Zanity","email":"sanfordmullen@zanity.com","city":"Martinez","state":"TX","_is_real":true} +{"index":{"_id":"786"}} +{"account_number":786,"balance":3024,"firstname":"Rene","lastname":"Vang","age":33,"gender":"M","address":"506 Randolph Street","employer":"Isopop","email":"renevang@isopop.com","city":"Vienna","state":"NJ","_is_real":true} +{"index":{"_id":"793"}} +{"account_number":793,"balance":16911,"firstname":"Alford","lastname":"Compton","age":36,"gender":"M","address":"186 Veronica Place","employer":"Zyple","email":"alfordcompton@zyple.com","city":"Sugartown","state":"AK","_is_real":true} +{"index":{"_id":"798"}} +{"account_number":798,"balance":3165,"firstname":"Catherine","lastname":"Ward","age":30,"gender":"F","address":"325 Burnett Street","employer":"Dreamia","email":"catherineward@dreamia.com","city":"Glenbrook","state":"SD","_is_real":true} +{"index":{"_id":"801"}} +{"account_number":801,"balance":14954,"firstname":"Molly","lastname":"Maldonado","age":37,"gender":"M","address":"518 Maple Avenue","employer":"Straloy","email":"mollymaldonado@straloy.com","city":"Hebron","state":"WI","_is_real":true} +{"index":{"_id":"806"}} +{"account_number":806,"balance":36492,"firstname":"Carson","lastname":"Riddle","age":31,"gender":"M","address":"984 Lois Avenue","employer":"Terrago","email":"carsonriddle@terrago.com","city":"Leland","state":"MN","_is_real":true} +{"index":{"_id":"813"}} +{"account_number":813,"balance":30833,"firstname":"Ebony","lastname":"Bishop","age":20,"gender":"M","address":"487 Ridge Court","employer":"Optique","email":"ebonybishop@optique.com","city":"Fairmount","state":"WA","_is_real":true} +{"index":{"_id":"818"}} +{"account_number":818,"balance":24433,"firstname":"Espinoza","lastname":"Petersen","age":26,"gender":"M","address":"641 Glenwood Road","employer":"Futurity","email":"espinozapetersen@futurity.com","city":"Floriston","state":"MD","_is_real":true} +{"index":{"_id":"820"}} +{"account_number":820,"balance":1011,"firstname":"Shepard","lastname":"Ramsey","age":24,"gender":"F","address":"806 Village Court","employer":"Mantro","email":"shepardramsey@mantro.com","city":"Tibbie","state":"NV","_is_real":true} +{"index":{"_id":"825"}} +{"account_number":825,"balance":49000,"firstname":"Terra","lastname":"Witt","age":21,"gender":"F","address":"590 Conway Street","employer":"Insectus","email":"terrawitt@insectus.com","city":"Forbestown","state":"AR","_is_real":true} +{"index":{"_id":"832"}} +{"account_number":832,"balance":8582,"firstname":"Laura","lastname":"Gibbs","age":39,"gender":"F","address":"511 Osborn Street","employer":"Corepan","email":"lauragibbs@corepan.com","city":"Worcester","state":"KS","_is_real":true} +{"index":{"_id":"837"}} +{"account_number":837,"balance":14485,"firstname":"Amy","lastname":"Villarreal","age":35,"gender":"M","address":"381 Stillwell Place","employer":"Fleetmix","email":"amyvillarreal@fleetmix.com","city":"Sanford","state":"IA","_is_real":true} +{"index":{"_id":"844"}} +{"account_number":844,"balance":26840,"firstname":"Jill","lastname":"David","age":31,"gender":"M","address":"346 Legion Street","employer":"Zytrax","email":"jilldavid@zytrax.com","city":"Saticoy","state":"SC","_is_real":true} +{"index":{"_id":"849"}} +{"account_number":849,"balance":16200,"firstname":"Barry","lastname":"Chapman","age":26,"gender":"M","address":"931 Dekoven Court","employer":"Darwinium","email":"barrychapman@darwinium.com","city":"Whitestone","state":"WY","_is_real":true} +{"index":{"_id":"851"}} +{"account_number":851,"balance":22026,"firstname":"Henderson","lastname":"Price","age":33,"gender":"F","address":"530 Hausman Street","employer":"Plutorque","email":"hendersonprice@plutorque.com","city":"Brutus","state":"RI","_is_real":true} +{"index":{"_id":"856"}} +{"account_number":856,"balance":27583,"firstname":"Alissa","lastname":"Knox","age":25,"gender":"M","address":"258 Empire Boulevard","employer":"Geologix","email":"alissaknox@geologix.com","city":"Hartsville/Hartley","state":"MN","_is_real":true} +{"index":{"_id":"863"}} +{"account_number":863,"balance":23165,"firstname":"Melendez","lastname":"Fernandez","age":40,"gender":"M","address":"661 Johnson Avenue","employer":"Vixo","email":"melendezfernandez@vixo.com","city":"Farmers","state":"IL","_is_real":true} +{"index":{"_id":"868"}} +{"account_number":868,"balance":27624,"firstname":"Polly","lastname":"Barron","age":22,"gender":"M","address":"129 Frank Court","employer":"Geofarm","email":"pollybarron@geofarm.com","city":"Loyalhanna","state":"ND","_is_real":true} +{"index":{"_id":"870"}} +{"account_number":870,"balance":43882,"firstname":"Goff","lastname":"Phelps","age":21,"gender":"M","address":"164 Montague Street","employer":"Digigen","email":"goffphelps@digigen.com","city":"Weedville","state":"IL","_is_real":true} +{"index":{"_id":"875"}} +{"account_number":875,"balance":19655,"firstname":"Mercer","lastname":"Pratt","age":24,"gender":"M","address":"608 Perry Place","employer":"Twiggery","email":"mercerpratt@twiggery.com","city":"Eggertsville","state":"MO","_is_real":true} +{"index":{"_id":"882"}} +{"account_number":882,"balance":10895,"firstname":"Mari","lastname":"Landry","age":39,"gender":"M","address":"963 Gerald Court","employer":"Kenegy","email":"marilandry@kenegy.com","city":"Lithium","state":"NC","_is_real":true} +{"index":{"_id":"887"}} +{"account_number":887,"balance":31772,"firstname":"Eunice","lastname":"Watts","age":36,"gender":"F","address":"707 Stuyvesant Avenue","employer":"Memora","email":"eunicewatts@memora.com","city":"Westwood","state":"TN","_is_real":true} +{"index":{"_id":"894"}} +{"account_number":894,"balance":1031,"firstname":"Tyler","lastname":"Fitzgerald","age":32,"gender":"M","address":"787 Meserole Street","employer":"Jetsilk","email":"tylerfitzgerald@jetsilk.com","city":"Woodlands","state":"WV","_is_real":true} +{"index":{"_id":"899"}} +{"account_number":899,"balance":32953,"firstname":"Carney","lastname":"Callahan","age":23,"gender":"M","address":"724 Kimball Street","employer":"Mangelica","email":"carneycallahan@mangelica.com","city":"Tecolotito","state":"MT","_is_real":true} +{"index":{"_id":"902"}} +{"account_number":902,"balance":13345,"firstname":"Hallie","lastname":"Jarvis","age":23,"gender":"F","address":"237 Duryea Court","employer":"Anixang","email":"halliejarvis@anixang.com","city":"Boykin","state":"IN","_is_real":true} +{"index":{"_id":"907"}} +{"account_number":907,"balance":12961,"firstname":"Ingram","lastname":"William","age":36,"gender":"M","address":"826 Overbaugh Place","employer":"Genmex","email":"ingramwilliam@genmex.com","city":"Kimmell","state":"AK","_is_real":true} +{"index":{"_id":"914"}} +{"account_number":914,"balance":7120,"firstname":"Esther","lastname":"Bean","age":32,"gender":"F","address":"583 Macon Street","employer":"Applica","email":"estherbean@applica.com","city":"Homeworth","state":"MN","_is_real":true} +{"index":{"_id":"919"}} +{"account_number":919,"balance":39655,"firstname":"盛虹","lastname":"Hanson","age":27,"gender":"M","address":"557 Hart Place","employer":"Exospace","email":"shaunahanson@exospace.com","city":"Outlook","state":"LA","_is_real":true} +{"index":{"_id":"921"}} +{"account_number":921,"balance":49119,"firstname":"Barbara","lastname":"Wade","age":29,"gender":"M","address":"687 Hoyts Lane","employer":"Roughies","email":"barbarawade@roughies.com","city":"Sattley","state":"CO","_is_real":true} +{"index":{"_id":"926"}} +{"account_number":926,"balance":49433,"firstname":"Welch","lastname":"Mcgowan","age":21,"gender":"M","address":"833 Quincy Street","employer":"Atomica","email":"welchmcgowan@atomica.com","city":"Hampstead","state":"VT","_is_real":true} +{"index":{"_id":"933"}} +{"account_number":933,"balance":18071,"firstname":"Tabitha","lastname":"Cole","age":21,"gender":"F","address":"916 Rogers Avenue","employer":"Eclipto","email":"tabithacole@eclipto.com","city":"Lawrence","state":"TX","_is_real":true} +{"index":{"_id":"938"}} +{"account_number":938,"balance":9597,"firstname":"Sharron","lastname":"Santos","age":40,"gender":"F","address":"215 Matthews Place","employer":"Zenco","email":"sharronsantos@zenco.com","city":"Wattsville","state":"VT","_is_real":true} +{"index":{"_id":"940"}} +{"account_number":940,"balance":23285,"firstname":"Melinda","lastname":"Mendoza","age":38,"gender":"M","address":"806 Kossuth Place","employer":"Kneedles","email":"melindamendoza@kneedles.com","city":"Coaldale","state":"OK","_is_real":true} +{"index":{"_id":"945"}} +{"account_number":945,"balance":23085,"firstname":"Hansen","lastname":"Hebert","age":33,"gender":"F","address":"287 Conduit Boulevard","employer":"Capscreen","email":"hansenhebert@capscreen.com","city":"Taycheedah","state":"AK","_is_real":true} +{"index":{"_id":"952"}} +{"account_number":952,"balance":21430,"firstname":"Angelique","lastname":"Weeks","age":33,"gender":"M","address":"659 Reeve Place","employer":"Exodoc","email":"angeliqueweeks@exodoc.com","city":"Turpin","state":"MD","_is_real":true} +{"index":{"_id":"957"}} +{"account_number":957,"balance":11373,"firstname":"Michael","lastname":"Giles","age":31,"gender":"M","address":"668 Court Square","employer":"Yogasm","email":"michaelgiles@yogasm.com","city":"Rosburg","state":"WV","_is_real":true} +{"index":{"_id":"964"}} +{"account_number":964,"balance":26154,"firstname":"Elena","lastname":"Waller","age":34,"gender":"F","address":"618 Crystal Street","employer":"Insurety","email":"elenawaller@insurety.com","city":"Gallina","state":"NY","_is_real":true} +{"index":{"_id":"969"}} +{"account_number":969,"balance":22214,"firstname":"Briggs","lastname":"Lynn","age":30,"gender":"M","address":"952 Lester Court","employer":"Quinex","email":"briggslynn@quinex.com","city":"Roland","state":"ID","_is_real":true} +{"index":{"_id":"971"}} +{"account_number":971,"balance":22772,"firstname":"Gabrielle","lastname":"Reilly","age":32,"gender":"F","address":"964 Tudor Terrace","employer":"Blanet","email":"gabriellereilly@blanet.com","city":"Falmouth","state":"AL","_is_real":true} +{"index":{"_id":"976"}} +{"account_number":976,"balance":31707,"firstname":"Mullen","lastname":"Tanner","age":26,"gender":"M","address":"711 Whitney Avenue","employer":"Pulze","email":"mullentanner@pulze.com","city":"Mooresburg","state":"MA","_is_real":true} +{"index":{"_id":"983"}} +{"account_number":983,"balance":47205,"firstname":"Mattie","lastname":"Eaton","age":24,"gender":"F","address":"418 Allen Avenue","employer":"Trasola","email":"mattieeaton@trasola.com","city":"Dupuyer","state":"NJ","_is_real":true} +{"index":{"_id":"988"}} +{"account_number":988,"balance":17803,"firstname":"Lucy","lastname":"Castro","age":34,"gender":"F","address":"425 Fleet Walk","employer":"Geekfarm","email":"lucycastro@geekfarm.com","city":"Mulino","state":"VA","_is_real":true} +{"index":{"_id":"990"}} +{"account_number":990,"balance":44456,"firstname":"Kelly","lastname":"Steele","age":35,"gender":"M","address":"809 Hoyt Street","employer":"Eschoir","email":"kellysteele@eschoir.com","city":"Stewartville","state":"ID","_is_real":true} +{"index":{"_id":"995"}} +{"account_number":995,"balance":21153,"firstname":"Phelps","lastname":"Parrish","age":25,"gender":"M","address":"666 Miller Place","employer":"Pearlessa","email":"phelpsparrish@pearlessa.com","city":"Brecon","state":"ME","_is_real":true} +{"index":{"_id":"9001"}} +{"account_number":9001,"balance":1000,"firstname":"Fake","lastname":"Alpha","age":25,"gender":"M","address":"1 Fake St","city":"Faketown","state":"CA","_is_real":false} +{"index":{"_id":"9002"}} +{"account_number":9002,"balance":2000,"firstname":"Fake","lastname":"Beta","age":30,"gender":"F","address":"2 Fake Ave","city":"Fakeville","state":"NY","_is_real":false} +{"index":{"_id":"9003"}} +{"account_number":9003,"balance":3000,"firstname":"Fake","lastname":"Gamma","age":35,"gender":"M","address":"3 Fake Blvd","city":"Fakeburg","state":"TX","_is_real":false} diff --git a/integ-test/src/test/resources/bank_extended.json b/integ-test/src/test/resources/bank_extended.json new file mode 100644 index 00000000000..c62744996c7 --- /dev/null +++ b/integ-test/src/test/resources/bank_extended.json @@ -0,0 +1,20 @@ +{"index":{"_id":"1"}} +{"account_number":1,"balance":39225,"firstname":"Amber JOHnny","lastname":"Duke Willmington","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL","male":true,"birthdate":"2017-10-23","_is_real":true} +{"index":{"_id":"6"}} +{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN","male":true,"birthdate":"2017-11-20","_is_real":true} +{"index":{"_id":"13"}} +{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA","male":false,"birthdate":"2018-06-23","_is_real":true} +{"index":{"_id":"18"}} +{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":"Boink","email":"daleadams@boink.com","city":"Orick","state":"MD","male":true,"birthdate":1542152000000,"_is_real":true} +{"index":{"_id":"20"}} +{"account_number":20,"balance":16418,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place","employer":"Scentric","email":"elinorratliff@scentric.com","city":"Ribera","state":"WA","male":true,"birthdate":"2018-06-27","_is_real":true} +{"index":{"_id":"25"}} +{"account_number":25,"balance":40540,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue","employer":"Filodyne","email":"virginiaayala@filodyne.com","city":"Nicholson","state":"PA","male":false,"birthdate":"2018-08-19","_is_real":true} +{"index":{"_id":"32"}} +{"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street","employer":"Quailcom","email":"dillardmcpherson@quailcom.com","city":"Veguita","state":"IN","male":false,"birthdate":"2018-08-11","_is_real":true} +{"index":{"_id":"99"}} +{"account_number":99,"balance":1000,"firstname":"Fake","lastname":"One","age":25,"gender":"M","address":"1 Fake Street","employer":"Fakecorp","email":"fakeone@fakecorp.com","city":"Faketown","state":"CA","male":true,"birthdate":"2020-01-01","_is_real":false} +{"index":{"_id":"100"}} +{"account_number":100,"balance":2000,"firstname":"Fake","lastname":"Two","age":30,"gender":"F","address":"2 Fake Avenue","employer":"Fakecorp","email":"faketwo@fakecorp.com","city":"Fakeville","state":"NY","male":false,"birthdate":"2020-06-15","_is_real":false} +{"index":{"_id":"101"}} +{"account_number":101,"balance":3000,"firstname":"Fake","lastname":"Three","age":35,"gender":"M","address":"3 Fake Boulevard","employer":"Fakecorp","email":"fakethree@fakecorp.com","city":"Fakeburg","state":"TX","male":true,"birthdate":"2021-03-10","_is_real":false} diff --git a/integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json new file mode 100644 index 00000000000..67b26a7c8f7 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json @@ -0,0 +1,53 @@ +{ + "mappings": { + "properties": { + "gender": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "address": { + "type": "text", + "fielddata": true + }, + "firstname": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "_is_real": { + "type": "boolean" + }, + "lastname": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json new file mode 100644 index 00000000000..2512751d9d2 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json @@ -0,0 +1,61 @@ +{ + "mappings": { + "properties": { + "account_number": { + "type": "long" + }, + "address": { + "type": "text" + }, + "age": { + "type": "integer" + }, + "balance": { + "type": "long" + }, + "birthdate": { + "type": "date" + }, + "city": { + "type": "keyword" + }, + "email": { + "type": "text" + }, + "employer": { + "type": "text" + }, + "firstname": { + "type": "keyword" + }, + "gender": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "_is_real": { + "type": "boolean" + }, + "lastname": { + "type": "keyword" + }, + "male": { + "type": "boolean" + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} From 804d4f1d5b4a846b84a010c86f652ed03968daae Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Tue, 9 Jun 2026 17:09:06 -0700 Subject: [PATCH 22/36] [BugFix] Fix SHOW/DESCRIBE statement routing under cluster.pluggable.dataformat setting (#5528) When cluster.pluggable.dataformat=composite, isAnalyticsIndex routed every query to the analytics engine, which cannot serve the system catalog (*_ODFE_SYS_TABLE, .DATASOURCES) that SHOW/DESCRIBE resolve. Detect system-catalog queries (including legacy-syntax SHOW/DESCRIBE that the V2 parser rejects) and keep them on the default pipeline while data queries continue to the analytics engine. Also log query routing to the analytics engine at both call sites. Signed-off-by: Chen Dai --- .../org/opensearch/sql/plugin/SQLPlugin.java | 2 + .../plugin/rest/RestUnifiedQueryAction.java | 24 +++++- .../transport/TransportPPLQueryAction.java | 5 ++ .../rest/RestUnifiedQueryActionTest.java | 78 ++++++++++++++++++- 4 files changed, 106 insertions(+), 3 deletions(-) diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 0eee03102bb..ab8b923e3ae 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -63,6 +63,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelperImpl; @@ -241,6 +242,7 @@ private BiFunction createSqlAnalyticsRout || !unifiedQueryHandler.isAnalyticsIndex(sqlRequest.getQuery(), QueryType.SQL)) { return false; } + LOGGER.info("[{}] Routing SQL query to analytics engine", QueryContext.getRequestId()); if (sqlRequest.isExplainRequest()) { unifiedQueryHandler.explain( sqlRequest.getQuery(), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 842dd2f1315..d81ce7b5137 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -13,6 +13,7 @@ import java.util.Map; import java.util.Optional; import org.apache.calcite.rel.RelNode; +import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.ThreadContext; @@ -40,6 +41,7 @@ import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import org.opensearch.sql.utils.SystemIndexUtils; import org.opensearch.transport.client.node.NodeClient; /** @@ -95,7 +97,17 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - return true; + // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + try (UnifiedQueryContext context = buildParsingContext(queryType)) { + boolean systemCatalog = + extractIndexName(query, queryType, context) + .map(RestUnifiedQueryAction::isSystemCatalog) + .orElse(false); + return !systemCatalog; + } catch (Exception e) { + // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. + return !isLegacySystemCatalogQuery(query); + } } try (UnifiedQueryContext context = buildParsingContext(queryType)) { return extractIndexName(query, queryType, context) @@ -107,6 +119,16 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { } } + private static boolean isSystemCatalog(String name) { + return SystemIndexUtils.isSystemIndex(name) + || SystemIndexUtils.DATASOURCES_TABLE_NAME.equals(name); + } + + private static boolean isLegacySystemCatalogQuery(String query) { + String trimmed = query.trim(); + return Strings.CI.startsWith(trimmed, "SHOW ") || Strings.CI.startsWith(trimmed, "DESCRIBE "); + } + private String stripSchemaPrefix(String indexName) { int lastDot = indexName.lastIndexOf('.'); return lastDot >= 0 ? indexName.substring(lastDot + 1) : indexName; diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index b7c2d2c9e11..fd231c29076 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -14,6 +14,8 @@ import java.util.Optional; import java.util.function.Supplier; import org.apache.calcite.rel.RelNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -59,6 +61,8 @@ public class TransportPPLQueryAction extends HandledTransportAction { + private static final Logger LOG = LogManager.getLogger(TransportPPLQueryAction.class); + private final Injector injector; private final Supplier pplEnabled; @@ -171,6 +175,7 @@ protected void doExecute( // Route to analytics engine for non-Lucene (e.g., Parquet-backed) indices. if (unifiedQueryHandler != null && unifiedQueryHandler.isAnalyticsIndex(transformedRequest.getRequest(), QueryType.PPL)) { + LOG.info("[{}] Routing PPL query to analytics engine", QueryContext.getRequestId()); if (transformedRequest.isExplainRequest()) { unifiedQueryHandler.explain( transformedRequest.getRequest(), diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 84b75161bd0..111597bb587 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -21,6 +21,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.IndicesService; import org.opensearch.sql.executor.QueryType; import org.opensearch.transport.client.node.NodeClient; @@ -142,7 +143,7 @@ public void nullAndEmptyQueriesRouteToLucene() { } @Test - public void showStatementRoutesToLucene() { + public void showStatementNotRoutedToAnalyticsEngine() { registerIndex( "parquet_logs", Settings.builder() @@ -154,7 +155,7 @@ public void showStatementRoutesToLucene() { } @Test - public void describeStatementRoutesToLucene() { + public void describeStatementNotRoutedToAnalyticsEngine() { registerIndex( "parquet_logs", Settings.builder() @@ -165,6 +166,79 @@ public void describeStatementRoutesToLucene() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void showStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("SHOW TABLES LIKE 'parquet_logs'", QueryType.SQL)); + } + + @Test + public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); + } + + @Test + public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + assertTrue(action.isAnalyticsIndex("SELECT * FROM parquet_logs", QueryType.SQL)); + } + + @Test + public void unparseableQueryRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + // malformed -> AE re-parses & reports + assertTrue(action.isAnalyticsIndex("SELECT FROM WHERE", QueryType.SQL)); + } + + @Test + public void legacyShowNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + // unquoted LIKE is rejected by the V2 parser, but still belongs on the default pipeline + assertFalse(action.isAnalyticsIndex("SHOW TABLES LIKE %", QueryType.SQL)); + } + + @Test + public void legacyDescribeNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + // legacy DESCRIBE syntax is rejected by the V2 parser, but belongs on the default pipeline + assertFalse(action.isAnalyticsIndex("DESCRIBE my_index", QueryType.SQL)); + } + + @Test + public void pplDescribeNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("describe parquet_logs", QueryType.PPL)); + } + + @Test + public void pplShowDatasourcesNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("show datasources", QueryType.PPL)); + } + + @Test + public void pplDataQueryStillRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + assertTrue(action.isAnalyticsIndex("source = parquet_logs | fields ts", QueryType.PPL)); + } + + @Test + public void pplUnparseableQueryRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + // malformed -> AE re-parses & reports + assertTrue(action.isAnalyticsIndex("source = parquet_logs | | fields ts", QueryType.PPL)); + } + + private void enableClusterComposite() { + when(clusterService.getSettings()) + .thenReturn( + Settings.builder() + .put( + IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "composite") + .build()); + } + private void registerIndex(String name, Settings settings) { IndexMetadata indexMetadata = mock(IndexMetadata.class); when(indexMetadata.getSettings()).thenReturn(settings); From 90358dcde89d348754906ee6b0cd810428fe595c Mon Sep 17 00:00:00 2001 From: Kai Huang <105710027+ahkcs@users.noreply.github.com> Date: Wed, 10 Jun 2026 13:27:08 -0700 Subject: [PATCH 23/36] Stabilize order-dependent PPL ITs with explicit sort for multi-shard analytics runs (#5537) The analytics engine merges per-shard Arrow batches in arrival order (no shard-ordinal tiebreaker), so any IT whose asserted rows depend on document encounter order fails when test indices have more than one primary shard. Add an explicit sort on a unique key to the affected queries so results are deterministic on every route; where the sorted result differs from insertion order (accounts.json is not account_number-ascending), update expectations to the sorted rows. Also add the tests.analytics.num_shards system property (default 1, wired through integTestRemote and TestUtils.AnalyticsIndexConfig) so multi-shard coverage runs can be reproduced: ./gradlew :integ-test:integTestRemote \ -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9300 \ -Dtests.clustername=runTask \ -Dtests.analytics.parquet_indices=true -Dtests.analytics.num_shards=3 Verified on three routes: 3-shard analytics, 1-shard analytics, and the regular Calcite route (self-managed cluster, all green). Signed-off-by: Kai Huang --- integ-test/build.gradle | 6 ++ .../calcite/remote/CalciteBinCommandIT.java | 12 ++-- .../remote/CalciteMVAppendFunctionIT.java | 10 +-- .../remote/CalciteMultisearchCommandIT.java | 8 ++- .../remote/CalcitePPLAggregationIT.java | 16 ++--- .../remote/CalcitePPLEnhancedCoalesceIT.java | 28 +++++---- .../sql/calcite/remote/CalcitePPLGrokIT.java | 10 ++- .../calcite/remote/CalcitePPLPatternsIT.java | 9 +-- .../calcite/remote/CalcitePPLTrendlineIT.java | 23 +++---- .../org/opensearch/sql/legacy/TestUtils.java | 12 +++- .../org/opensearch/sql/ppl/HeadCommandIT.java | 61 +++++++++++-------- .../opensearch/sql/ppl/TrailingPipeIT.java | 34 +++++++---- 12 files changed, 139 insertions(+), 90 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 6554b7c8491..c8515f015a2 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1035,6 +1035,12 @@ task integTestRemote(type: RestIntegTestTask) { systemProperty 'tests.analytics.parquet_indices', System.getProperty("tests.analytics.parquet_indices") } + // Primary-shard count for analytics-backed test indices (default 1). Set to e.g. 3 to + // reproduce multi-shard coordination behavior on a single-node cluster. + if (System.getProperty("tests.analytics.num_shards") != null) { + systemProperty 'tests.analytics.num_shards', System.getProperty("tests.analytics.num_shards") + } + // True only when the analytics-engine route is active (every test index parquet-backed). Matches // AnalyticsIndexConfig.isEnabled, which parses the value rather than checking mere presence, so a // `-Dtests.analytics.parquet_indices=false` run stays on the v2 path. diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java index 8bcfd034a8d..3fb8219fe12 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java @@ -85,10 +85,12 @@ public void testBinWithMinspan() throws IOException { public void testBinBasicFunctionality() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | bin age span=5 | fields age | head 3", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | bin age span=5 | sort account_number | fields age | head 3", + TEST_INDEX_ACCOUNT)); verifySchema(result, schema("age", null, "string")); - verifyDataRows(result, rows("30-35"), rows("35-40"), rows("25-30")); + verifyDataRows(result, rows("25-30"), rows("30-35"), rows("20-25")); } @Test @@ -195,7 +197,9 @@ public void testBinOnlyWithoutAggregation() throws IOException { JSONObject binOnlyResult = executeQuery( String.format( - "source=%s" + " | bin @timestamp span=4h" + " | fields `@timestamp` | head 3", + "source=%s" + + " | bin @timestamp span=4h" + + " | fields `@timestamp` | sort `@timestamp` | head 3", TEST_INDEX_TIME_DATA)); // Verify schema and that binning works correctly @@ -235,7 +239,7 @@ public void testBinWithMonthlySpan() throws IOException { executeQuery( String.format( "source=%s | bin @timestamp span=4mon as cate | fields" - + " cate, @timestamp | head 5", + + " cate, @timestamp | sort @timestamp | head 5", TEST_INDEX_TIME_DATA)); verifySchema(result, schema("cate", null, "string"), schema("@timestamp", null, "timestamp")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java index 9afb63056e8..546adc4755b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java @@ -107,8 +107,8 @@ public void testMvappendWithRealFields() throws IOException { executeQuery( source( TEST_INDEX_BANK, - "eval result = mvappend(firstname, lastname) | head 1 | fields firstname, lastname," - + " result")); + "eval result = mvappend(firstname, lastname) | sort account_number | head 1 |" + + " fields firstname, lastname, result")); verifySchema( actual, @@ -179,7 +179,7 @@ public void testMvappendInWhereClause() throws IOException { source( TEST_INDEX_BANK, "eval combined = mvappend(firstname, lastname) | where array_length(combined) = 2 |" - + " head 1 | fields firstname, lastname, combined")); + + " sort account_number | head 1 | fields firstname, lastname, combined")); verifySchema( actual, @@ -198,8 +198,8 @@ public void testMvappendWithComplexExpression() throws IOException { executeQuery( source( TEST_INDEX_BANK, - "eval result = mvappend(array(age), array(age * 2), age + 10) | head 1 | fields" - + " age, result")); + "eval result = mvappend(array(age), array(age * 2), age + 10) | sort" + + " account_number | head 1 | fields age, result")); verifySchema(actual, schema("age", "int"), schema("result", "array")); verifyDataRows(actual, rows(32, List.of(32, 64, 42))); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java index d0a4882fb1f..a738132744b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java @@ -325,8 +325,10 @@ public void testMultisearchCrossIndexFieldSelection() throws IOException { executeQuery( String.format( "| multisearch " - + "[search source=%s | fields firstname, balance | head 2] " - + "[search source=%s | fields description, place_id | head 2]", + + "[search source=%s | sort account_number | fields firstname, balance" + + " | head 2] " + + "[search source=%s | sort place_id | fields description, place_id" + + " | head 2]", TEST_INDEX_ACCOUNT, TEST_INDEX_LOCATIONS_TYPE_CONFLICT)); verifySchema( @@ -338,8 +340,8 @@ public void testMultisearchCrossIndexFieldSelection() throws IOException { verifyDataRows( result, + rows("Bradshaw", 16623L, null, null), rows("Amber", 39225L, null, null), - rows("Hattie", 5686L, null, null), rows(null, null, "Central Park", 1001), rows(null, null, "Times Square", 1002)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index 5c8f6eb2cf7..a5937d06f31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -542,8 +542,8 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 15minute) as" - + " datetime_span", + "source=%s | sort key | head 5 | stats count(datetime0) by span(datetime0," + + " 15minute) as datetime_span", TEST_INDEX_CALCS)); verifySchema( actual, schema("datetime_span", "timestamp"), schema("count(datetime0)", "bigint")); @@ -558,8 +558,8 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 5second) as" - + " datetime_span", + "source=%s | sort key | head 5 | stats count(datetime0) by span(datetime0," + + " 5second) as datetime_span", TEST_INDEX_CALCS)); verifySchema( actual, schema("datetime_span", "timestamp"), schema("count(datetime0)", "bigint")); @@ -574,8 +574,8 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 3month) as" - + " datetime_span", + "source=%s | sort key | head 5 | stats count(datetime0) by span(datetime0," + + " 3month) as datetime_span", TEST_INDEX_CALCS)); verifySchema( actual, schema("datetime_span", "timestamp"), schema("count(datetime0)", "bigint")); @@ -587,8 +587,8 @@ public void testCountByNullableTimeSpan() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0), count(datetime1) by span(time1," - + " 15minute) as time_span", + "source=%s | sort key | head 5 | stats count(datetime0), count(datetime1) by" + + " span(time1, 15minute) as time_span", TEST_INDEX_CALCS)); verifySchema( actual, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java index fd9a5cff774..a0e6c2679fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java @@ -73,8 +73,8 @@ public void testCoalesceWithLiterals() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(name, 123, 'unknown') | fields name, result |" - + " head 1", + "source=%s | eval result = coalesce(name, 123, 'unknown') | sort - age | fields" + + " name, result | head 1", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -100,8 +100,8 @@ public void testCoalesceWithMultipleFields() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(name, age, year, month) | fields name, age," - + " year, month, result | head 2", + "source=%s | eval result = coalesce(name, age, year, month) | sort - age | fields" + + " name, age, year, month, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema( @@ -139,8 +139,8 @@ public void testCoalesceWithNonExistentField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(nonexistent_field, name) | fields name, result" - + " | head 2", + "source=%s | eval result = coalesce(nonexistent_field, name) | sort - age" + + " | fields name, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -234,7 +234,8 @@ public void testCoalesceWithEmptyString() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce('', name) | fields name, result | head 1", + "source=%s | eval result = coalesce('', name) | sort - age | fields name, result" + + " | head 1", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -247,7 +248,8 @@ public void testCoalesceWithSpaceString() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(' ', name) | fields name, result | head 1", + "source=%s | eval result = coalesce(' ', name) | sort - age | fields name, result" + + " | head 1", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -274,8 +276,8 @@ public void testCoalesceWithCompatibleNumericTypes() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(age, year, 999) | fields age, year, result |" - + " head 2", + "source=%s | eval result = coalesce(age, year, 999) | sort - age | fields age," + + " year, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("age", "int"), schema("year", "int"), schema("result", "int")); @@ -288,7 +290,7 @@ public void testCoalesceTypeCoercionWithMixedTypes() throws IOException { executeQuery( String.format( "source=%s | eval result = coalesce(nonexistent_field, age," - + " 'default') | fields age, result | head 2", + + " 'default') | sort - age | fields age, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("age", "int"), schema("result", "string")); @@ -300,8 +302,8 @@ public void testCoalesceWithCompatibleNumericAndTemporalTypes() throws IOExcepti JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(age, year, month) | fields age, year, month," - + " result | head 2", + "source=%s | eval result = coalesce(age, year, month) | sort - age | fields age," + + " year, month, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java index 72510e589a9..716eead0478 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java @@ -34,7 +34,8 @@ public void testGrokEmail() throws IOException { executeQuery( String.format( Locale.ROOT, - "source = %s | grok email '%s' | head 3 | fields email, host", + "source = %s | grok email '%s' | sort account_number | head 3 | fields email," + + " host", TEST_INDEX_BANK, ".+@%{HOSTNAME:host}")); verifySchema(result, schema("email", "string"), schema("host", "string")); @@ -49,7 +50,10 @@ public void testGrokEmail() throws IOException { public void testGrokAddressOverriding() throws IOException { JSONObject preGrokResult = executeQuery( - String.format(Locale.ROOT, "source = %s | head 3 | fields address", TEST_INDEX_BANK)); + String.format( + Locale.ROOT, + "source = %s | sort account_number | head 3 | fields address", + TEST_INDEX_BANK)); verifySchema(preGrokResult, schema("address", "string")); verifyDataRows( preGrokResult, @@ -61,7 +65,7 @@ public void testGrokAddressOverriding() throws IOException { executeQuery( String.format( Locale.ROOT, - "source = %s | grok address '%s' | head 3 | fields address", + "source = %s | grok address '%s' | sort account_number | head 3 | fields address", TEST_INDEX_BANK, "%{NUMBER} %{GREEDYDATA:address}")); verifySchema(result, schema("address", "string")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java index 46df914e611..fac69220fbf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java @@ -37,7 +37,8 @@ public void testSimplePatternLabelMode_NotShowNumberedToken() throws IOException JSONObject result = executeQuery( String.format( - "source = %s | patterns email mode=label | head 1 | fields email, patterns_field", + "source = %s | patterns email mode=label | sort account_number | head 1 | fields" + + " email, patterns_field", TEST_INDEX_BANK)); verifySchema(result, schema("email", "string"), schema("patterns_field", "string")); verifyDataRows(result, rows("amberduke@pyrami.com", "<*>@<*>.<*>")); @@ -48,8 +49,8 @@ public void testSimplePatternLabelMode_ShowNumberedToken() throws IOException { JSONObject result = executeQuery( String.format( - "source = %s | patterns email mode=label show_numbered_token=true | head 1 | fields" - + " email, patterns_field, tokens", + "source = %s | patterns email mode=label show_numbered_token=true | sort" + + " account_number | head 1 | fields email, patterns_field, tokens", TEST_INDEX_BANK)); verifySchema( result, @@ -100,7 +101,7 @@ public void testSimplePatternLabelModeWithCustomPattern_ShowNumberedToken() thro executeQuery( String.format( "source = %s | patterns email mode=label show_numbered_token=true pattern='@.*' |" - + " head 1 | fields email, patterns_field, tokens", + + " sort account_number | head 1 | fields email, patterns_field, tokens", TEST_INDEX_BANK)); verifySchema( result, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java index 8d31354aa90..47e28c3a827 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java @@ -33,8 +33,8 @@ public void testTrendlineSma() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(3, balance) as balance_trend |" - + " fields balance_trend", + "source=%s | where balance > 30000 | sort account_number | trendline sma(3," + + " balance) as balance_trend | fields balance_trend", TEST_INDEX_BANK)); verifySchema(result, schema("balance_trend", "double")); verifyDataRows( @@ -46,8 +46,8 @@ public void testTrendlineWma() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline wma(3, balance) as balance_trend |" - + " fields balance_trend", + "source=%s | where balance > 30000 | sort account_number | trendline wma(3," + + " balance) as balance_trend | fields balance_trend", TEST_INDEX_BANK)); verifySchema(result, schema("balance_trend", "double")); verifyDataRows( @@ -59,8 +59,8 @@ public void testTrendlineMultipleFields() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(2, balance) as sma wma(3," - + " balance) as wma | fields balance, sma, wma", + "source=%s | where balance > 30000 | sort account_number | trendline sma(2," + + " balance) as sma wma(3, balance) as wma | fields balance, sma, wma", TEST_INDEX_BANK)); verifySchema( result, schema("balance", "bigint"), schema("sma", "double"), schema("wma", "double")); @@ -77,8 +77,8 @@ public void testTrendlineNoAlias() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(2, balance) | fields" - + " balance, balance_trendline", + "source=%s | where balance > 30000 | sort account_number | trendline sma(2," + + " balance) | fields balance, balance_trendline", TEST_INDEX_BANK)); verifySchema(result, schema("balance", "bigint"), schema("balance_trendline", "double")); verifyDataRows( @@ -90,8 +90,8 @@ public void testTrendlineOverwritesExisingField() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(2, balance) as balance | fields" - + " balance", + "source=%s | where balance > 30000 | sort account_number | trendline sma(2," + + " balance) as balance | fields balance", TEST_INDEX_BANK)); verifySchema(result, schema("balance", "double")); verifyDataRows(result, rows((Object) null), rows(36031.5), rows(36689), rows(44313)); @@ -115,7 +115,8 @@ public void testTrendlinePreFilterNullValues() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | trendline sma(2, balance) | fields" + " balance, balance_trendline", + "source=%s | sort account_number | trendline sma(2, balance) | fields" + + " balance, balance_trendline", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(result, schema("balance", "bigint"), schema("balance_trendline", "double")); verifyDataRows( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index e3af81fc44a..41bee8fd0c6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -65,10 +65,20 @@ public static final class AnalyticsIndexConfig { */ public static final String ENABLED_PROP = "tests.analytics.parquet_indices"; + /** + * System property overriding the number of primary shards for analytics-backed test indices. + * Defaults to 1 (single-shard). Set to e.g. "3" for multi-shard coverage runs. + */ + public static final String NUM_SHARDS_PROP = "tests.analytics.num_shards"; + public static boolean isEnabled() { return Boolean.parseBoolean(System.getProperty(ENABLED_PROP, "false")); } + public static int getNumShards() { + return Integer.parseInt(System.getProperty(NUM_SHARDS_PROP, "1")); + } + // Composite-store format values shared by the index-level and cluster-level settings below. private static final String DATAFORMAT_COMPOSITE = "composite"; private static final String PRIMARY_FORMAT_PARQUET = "parquet"; @@ -86,7 +96,7 @@ static void applyIndexCreationSettings(JSONObject jsonObject) { jsonObject.has("settings") ? jsonObject.getJSONObject("settings") : new JSONObject(); JSONObject indexSettings = settings.has("index") ? settings.getJSONObject("index") : new JSONObject(); - indexSettings.put("number_of_shards", 1); + indexSettings.put("number_of_shards", getNumShards()); indexSettings.put("pluggable.dataformat.enabled", true); indexSettings.put("pluggable.dataformat", DATAFORMAT_COMPOSITE); indexSettings.put("composite.primary_data_format", PRIMARY_FORMAT_PARQUET); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java index a24896adaf3..06fb7b627f9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java @@ -38,27 +38,32 @@ public void init() throws Exception { @Test public void testHead() throws IOException { JSONObject result = - executeQuery(String.format("source=%s | fields firstname, age | head", TEST_INDEX_ACCOUNT)); + executeQuery( + String.format( + "source=%s | sort account_number | fields firstname, age | head", + TEST_INDEX_ACCOUNT)); verifyDataRows( result, + rows("Bradshaw", 29), rows("Amber", 32), + rows("Roberta", 22), + rows("Levine", 26), + rows("Rodriquez", 31), + rows("Leola", 30), rows("Hattie", 36), - rows("Nanette", 28), - rows("Dale", 33), - rows("Elinor", 36), - rows("Virginia", 39), - rows("Dillard", 34), - rows("Mcgee", 39), - rows("Aurelia", 37), - rows("Fulton", 23)); + rows("Levy", 22), + rows("Jan", 35), + rows("Opal", 39)); } @Test public void testHeadWithNumber() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | fields firstname, age | head 3", TEST_INDEX_ACCOUNT)); - verifyDataRows(result, rows("Amber", 32), rows("Hattie", 36), rows("Nanette", 28)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Bradshaw", 29), rows("Amber", 32), rows("Roberta", 22)); } @Ignore("Fix https://github.com/opensearch-project/sql/issues/703#issuecomment-1211422130") @@ -87,24 +92,26 @@ public void testHeadWithNumberLargerThanMaxResultWindow() throws IOException { setMaxResultWindow(TEST_INDEX_ACCOUNT, 10); JSONObject result = executeQuery( - String.format("source=%s | fields firstname, age | head 15", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 15", + TEST_INDEX_ACCOUNT)); verifyDataRows( result, + rows("Bradshaw", 29), rows("Amber", 32), + rows("Roberta", 22), + rows("Levine", 26), + rows("Rodriquez", 31), + rows("Leola", 30), rows("Hattie", 36), + rows("Levy", 22), + rows("Jan", 35), + rows("Opal", 39), + rows("Dominique", 37), + rows("Jenkins", 20), + rows("Stafford", 20), rows("Nanette", 28), - rows("Dale", 33), - rows("Elinor", 36), - rows("Virginia", 39), - rows("Dillard", 34), - rows("Mcgee", 39), - rows("Aurelia", 37), - rows("Fulton", 23), - rows("Burton", 31), - rows("Josie", 32), - rows("Hughes", 30), - rows("Hall", 25), - rows("Deidre", 33)); + rows("Erma", 39)); } @Ignore("Fix https://github.com/opensearch-project/sql/issues/703#issuecomment-1211422130") @@ -138,7 +145,9 @@ public void testHeadWithLargeNumber() throws IOException { public void testHeadWithNumberAndFrom() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | fields firstname, age | head 3 from 4", TEST_INDEX_ACCOUNT)); - verifyDataRows(result, rows("Elinor", 36), rows("Virginia", 39), rows("Dillard", 34)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3 from 4", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Rodriquez", 31), rows("Leola", 30), rows("Hattie", 36)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java index 8b5df6e710e..3d16bd034bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java @@ -34,8 +34,10 @@ public void init() throws Exception { @Test public void testTrailingPipeAfterSource() throws IOException { // Query with trailing pipe should produce same results as without - JSONObject resultWithout = executeQuery(String.format("source=%s", TEST_INDEX_ACCOUNT)); - JSONObject resultWith = executeQuery(String.format("source=%s |", TEST_INDEX_ACCOUNT)); + JSONObject resultWithout = + executeQuery(String.format("source=%s | sort account_number", TEST_INDEX_ACCOUNT)); + JSONObject resultWith = + executeQuery(String.format("source=%s | sort account_number |", TEST_INDEX_ACCOUNT)); // Both should return the same data assertTrue(resultWithout.similar(resultWith)); @@ -52,11 +54,13 @@ public void testTrailingPipeAfterFields() throws IOException { JSONObject resultWithout = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age", TEST_INDEX_ACCOUNT)); + "source=%s | where age > 30 | sort account_number | fields firstname, age", + TEST_INDEX_ACCOUNT)); JSONObject resultWith = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age |", TEST_INDEX_ACCOUNT)); + "source=%s | where age > 30 | sort account_number | fields firstname, age |", + TEST_INDEX_ACCOUNT)); assertTrue(resultWithout.similar(resultWith)); } @@ -71,10 +75,14 @@ public void testTrailingPipeAfterFields() throws IOException { public void testTrailingPipeAfterHead() throws IOException { JSONObject resultWithout = executeQuery( - String.format("source=%s | fields firstname, age | head 3", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3", + TEST_INDEX_ACCOUNT)); JSONObject resultWith = executeQuery( - String.format("source=%s | fields firstname, age | head 3 |", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3 |", + TEST_INDEX_ACCOUNT)); assertTrue(resultWithout.similar(resultWith)); } @@ -115,11 +123,13 @@ public void testEmptyPipeInMiddle() throws IOException { JSONObject resultNormal = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age", TEST_INDEX_ACCOUNT)); + "source=%s | where age > 30 | sort account_number | fields firstname, age", + TEST_INDEX_ACCOUNT)); JSONObject resultWithEmpty = executeQuery( String.format( - "source=%s | | where age > 30 | fields firstname, age", TEST_INDEX_ACCOUNT)); + "source=%s | | where age > 30 | sort account_number | fields firstname, age", + TEST_INDEX_ACCOUNT)); assertTrue(resultNormal.similar(resultWithEmpty)); } @@ -136,12 +146,12 @@ public void testMultipleEmptyPipes() throws IOException { JSONObject resultNormal = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age | sort age", + "source=%s | where age > 30 | sort age, account_number | fields firstname, age", TEST_INDEX_ACCOUNT)); JSONObject resultWithEmpty = executeQuery( String.format( - "source=%s | | where age > 30 | | fields firstname, age | sort age", + "source=%s | | where age > 30 | | sort age, account_number | fields firstname, age", TEST_INDEX_ACCOUNT)); assertTrue(resultNormal.similar(resultWithEmpty)); @@ -159,12 +169,12 @@ public void testEmptyPipesAndTrailingPipe() throws IOException { JSONObject resultNormal = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age | sort age", + "source=%s | where age > 30 | sort age, account_number | fields firstname, age", TEST_INDEX_ACCOUNT)); JSONObject resultWithEmpty = executeQuery( String.format( - "source=%s | | where age > 30 | fields firstname, age | sort age |", + "source=%s | | where age > 30 | sort age, account_number | fields firstname, age |", TEST_INDEX_ACCOUNT)); assertTrue(resultNormal.similar(resultWithEmpty)); From 8cce7a828f73c6c56e3a8c54e1f7347f9f50f06e Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 10 Jun 2026 13:48:24 -0700 Subject: [PATCH 24/36] update datetime tests to stay within ae bounds (#5534) Signed-off-by: Simeon Widdis --- .../sql/legacy/SQLIntegTestCase.java | 5 ++ .../org/opensearch/sql/legacy/TestUtils.java | 5 ++ .../opensearch/sql/legacy/TestsConstants.java | 1 + .../sql/ppl/DateTimeComparisonIT.java | 56 +++++++++---------- .../sql/sql/DateTimeComparisonIT.java | 51 ++++++++--------- .../src/test/resources/datetime_simple.json | 2 + .../datetime_simple_index_mapping.json | 10 ++++ 7 files changed, 77 insertions(+), 53 deletions(-) create mode 100644 integ-test/src/test/resources/datetime_simple.json create mode 100644 integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index bde077c20c3..183a060f7f0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -732,6 +732,11 @@ public enum Index { "_doc", getDataTypeNonnumericIndexMapping(), "src/test/resources/datatypes.json"), + DATETIME_SIMPLE( + TestsConstants.TEST_INDEX_DATETIME_SIMPLE, + "_doc", + getDateTimeSimpleIndexMapping(), + "src/test/resources/datetime_simple.json"), BEER( TestsConstants.TEST_INDEX_BEER, "beer", null, "src/test/resources/beer.stackexchange.json"), NULL_MISSING( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 41bee8fd0c6..f54f1c684fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -414,6 +414,11 @@ public static String getDataTypeNonnumericIndexMapping() { return getMappingFile(mappingFile); } + public static String getDateTimeSimpleIndexMapping() { + String mappingFile = "datetime_simple_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getGeopointIndexMapping() { String mappingFile = "geopoint_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index f78deb532c4..5d7eeb328af 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -61,6 +61,7 @@ public class TestsConstants { public static final String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; public static final String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; public static final String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; + public static final String TEST_INDEX_DATETIME_SIMPLE = TEST_INDEX + "_datetime_simple"; public static final String TEST_INDEX_BEER = TEST_INDEX + "_beer"; public static final String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; public static final String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java index 1cef2a43001..e31a73d91a6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java @@ -7,7 +7,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.$; import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATETIME_SIMPLE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -28,7 +28,7 @@ public class DateTimeComparisonIT extends PPLIntegTestCase { @Override public void init() throws Exception { super.init(); - loadIndex(Index.DATA_TYPE_NONNUMERIC); + loadIndex(Index.DATETIME_SIMPLE); } private final TimeZone testTz = TimeZone.getDefault(); @@ -231,7 +231,7 @@ public static Iterable compareLtTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') < DATE('1961-04-12')", "ts_d_f", false), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2020-09-16 10:20:30') < TIME('09:07:00')", "ts_t_t", true), - $("TIME('09:07:00') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('09:07:00') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') < TIME('10:20:30')", "ts_t_f", false), $("TIME('20:50:40') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -240,13 +240,13 @@ public static Iterable compareLtTimestampWithOtherTypes() { public static Iterable compareLtDateWithOtherTypes() { return Arrays.asList( $$( - $("DATE('2020-09-16') < TIMESTAMP('3077-04-12 09:07:00')", "d_ts_t", true), + $("DATE('2020-09-16') < TIMESTAMP('2242-04-12 09:07:00')", "d_ts_t", true), $("TIMESTAMP('1961-04-12 09:07:00') < DATE('1984-12-15')", "ts_d_t", true), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') < DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') < TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') < TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') < DATE('2020-09-16')", "t_d_f", false))); } @@ -255,14 +255,14 @@ public static Iterable compareLtTimeWithOtherTypes() { var today = LocalDate.now().toString(); return Arrays.asList( $$( - $("TIME('22:15:07') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('22:15:07') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('1984-12-15 10:20:30') < TIME('10:20:30')", "ts_t_t", true), $("TIME('10:20:30') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') < TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') < DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') < TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') < TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -274,10 +274,10 @@ public static Iterable compareGtTimestampWithOtherTypes() { $("DATE('2020-09-16') > TIMESTAMP('2020-09-15 22:15:07')", "d_ts_t", true), $("TIMESTAMP('2020-09-16 10:20:30') > DATE('2077-04-12')", "ts_d_f", false), $("DATE('1961-04-12') > TIMESTAMP('1961-04-12 00:00:00')", "d_ts_f", false), - $("TIMESTAMP('3077-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), + $("TIMESTAMP('2242-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') > TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') > TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') > TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -288,10 +288,10 @@ public static Iterable compareGtDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') > DATE('2020-09-16')", "ts_d_t", true), $("DATE('2020-09-16') > TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') > DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') > TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') > TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') > DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -304,8 +304,8 @@ public static Iterable compareGtTimeWithOtherTypes() { $("TIME('10:20:30') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') > TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') > DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') > TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') > TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false))); } @@ -319,8 +319,8 @@ public static Iterable compareLteTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') <= DATE('1961-04-12')", "ts_d_f", false), $("DATE('2077-04-12') <= TIMESTAMP('1984-12-15 22:15:07')", "d_ts_f", false), $("TIMESTAMP('" + today + " 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), - $("TIME('09:07:00') <= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), - $("TIMESTAMP('3077-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), + $("TIME('09:07:00') <= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), + $("TIMESTAMP('2242-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), $("TIME('20:50:40') <= TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -333,8 +333,8 @@ public static Iterable compareLteDateWithOtherTypes() { $("DATE('2020-09-16') <= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') <= DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') <= TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') <= TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') <= DATE('2020-09-16')", "t_d_f", false))); } @@ -347,10 +347,10 @@ public static Iterable compareLteTimeWithOtherTypes() { $("TIMESTAMP('1984-12-15 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), $("TIME('22:15:07') <= TIMESTAMP('1984-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') <= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') <= DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') <= TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') <= TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -365,7 +365,7 @@ public static Iterable compareGteTimestampWithOtherTypes() { $("TIMESTAMP('" + today + " 10:20:30') >= TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('1977-07-08 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -376,10 +376,10 @@ public static Iterable compareGteDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') >= DATE('2020-09-16')", "ts_d_t", true), $("DATE('1961-04-12') >= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') >= DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') >= TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') >= TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') >= DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -389,11 +389,11 @@ public static Iterable compareGteTimeWithOtherTypes() { $$( $("TIME('10:20:30') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 20:50:42') >= TIME('10:20:30')", "ts_t_t", true), - $("TIME('22:15:07') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false), + $("TIME('22:15:07') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') >= DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') >= TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') >= TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false))); } @@ -403,7 +403,7 @@ public void testCompare() throws IOException { executeQuery( String.format( "source=%s | eval `%s` = %s | fields `%s`", - TEST_INDEX_DATATYPE_NONNUMERIC, name, functionCall, name)); + TEST_INDEX_DATETIME_SIMPLE, name, functionCall, name)); verifySchema(result, schema(name, null, "boolean")); verifyDataRows(result, rows(expectedResult)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java index d385b54dff1..933fc7b3ed5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java @@ -35,6 +35,7 @@ public class DateTimeComparisonIT extends SQLIntegTestCase { @Override public void init() throws Exception { super.init(); + loadIndex(Index.DATETIME_SIMPLE); } private final TimeZone testTz = TimeZone.getDefault(); @@ -237,7 +238,7 @@ public static Iterable compareLtTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') < DATE('1961-04-12')", "ts_d_f", false), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2020-09-16 10:20:30') < TIME('09:07:00')", "ts_t_t", true), - $("TIME('09:07:00') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('09:07:00') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') < TIME('10:20:30')", "ts_t_f", false), $("TIME('20:50:40') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -246,13 +247,13 @@ public static Iterable compareLtTimestampWithOtherTypes() { public static Iterable compareLtDateWithOtherTypes() { return Arrays.asList( $$( - $("DATE('2020-09-16') < TIMESTAMP('3077-04-12 09:07:00')", "d_ts_t", true), + $("DATE('2020-09-16') < TIMESTAMP('2242-04-12 09:07:00')", "d_ts_t", true), $("TIMESTAMP('1961-04-12 09:07:00') < DATE('1984-12-15')", "ts_d_t", true), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') < DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') < TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') < TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') < DATE('2020-09-16')", "t_d_f", false))); } @@ -261,14 +262,14 @@ public static Iterable compareLtTimeWithOtherTypes() { var today = LocalDate.now().toString(); return Arrays.asList( $$( - $("TIME('22:15:07') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('22:15:07') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('1984-12-15 10:20:30') < TIME('10:20:30')", "ts_t_t", true), $("TIME('10:20:30') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') < TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') < DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') < TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') < TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -280,10 +281,10 @@ public static Iterable compareGtTimestampWithOtherTypes() { $("DATE('2020-09-16') > TIMESTAMP('2020-09-15 22:15:07')", "d_ts_t", true), $("TIMESTAMP('2020-09-16 10:20:30') > DATE('2077-04-12')", "ts_d_f", false), $("DATE('1961-04-12') > TIMESTAMP('1961-04-12 00:00:00')", "d_ts_f", false), - $("TIMESTAMP('3077-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), + $("TIMESTAMP('2242-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') > TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') > TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') > TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -294,10 +295,10 @@ public static Iterable compareGtDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') > DATE('2020-09-16')", "ts_d_t", true), $("DATE('2020-09-16') > TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') > DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') > TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') > TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') > DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -310,8 +311,8 @@ public static Iterable compareGtTimeWithOtherTypes() { $("TIME('10:20:30') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') > TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') > DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') > TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') > TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false))); } @@ -325,8 +326,8 @@ public static Iterable compareLteTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') <= DATE('1961-04-12')", "ts_d_f", false), $("DATE('2077-04-12') <= TIMESTAMP('1984-12-15 22:15:07')", "d_ts_f", false), $("TIMESTAMP('" + today + " 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), - $("TIME('09:07:00') <= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), - $("TIMESTAMP('3077-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), + $("TIME('09:07:00') <= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), + $("TIMESTAMP('2242-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), $("TIME('20:50:40') <= TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -339,8 +340,8 @@ public static Iterable compareLteDateWithOtherTypes() { $("DATE('2020-09-16') <= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') <= DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') <= TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') <= TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') <= DATE('2020-09-16')", "t_d_f", false))); } @@ -353,10 +354,10 @@ public static Iterable compareLteTimeWithOtherTypes() { $("TIMESTAMP('1984-12-15 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), $("TIME('22:15:07') <= TIMESTAMP('1984-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') <= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') <= DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') <= TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') <= TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -371,7 +372,7 @@ public static Iterable compareGteTimestampWithOtherTypes() { $("TIMESTAMP('" + today + " 10:20:30') >= TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('1977-07-08 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -382,10 +383,10 @@ public static Iterable compareGteDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') >= DATE('2020-09-16')", "ts_d_t", true), $("DATE('1961-04-12') >= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') >= DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') >= TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') >= TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') >= DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -395,11 +396,11 @@ public static Iterable compareGteTimeWithOtherTypes() { $$( $("TIME('10:20:30') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 20:50:42') >= TIME('10:20:30')", "ts_t_t", true), - $("TIME('22:15:07') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false), + $("TIME('22:15:07') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') >= DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') >= TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') >= TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false))); } diff --git a/integ-test/src/test/resources/datetime_simple.json b/integ-test/src/test/resources/datetime_simple.json new file mode 100644 index 00000000000..7ed96384a24 --- /dev/null +++ b/integ-test/src/test/resources/datetime_simple.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"date_value": "2020-10-13 13:00:00"} diff --git a/integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json new file mode 100644 index 00000000000..fa2283f68f7 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json @@ -0,0 +1,10 @@ +{ + "mappings": { + "properties": { + "date_value": { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" + } + } + } +} From 8394e5cedf198d31335502275a907e53a4281504 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 10 Jun 2026 14:09:33 -0700 Subject: [PATCH 25/36] fix(sql): Report invalid-query errors as client errors (#5532) UnifiedQueryPlanner.plan() let field-not-found (ErrorReport) and Calcite validation errors such as table-not-found (CalciteException) fall through to the catch-all and rethrew them as IllegalStateException. Propagate ErrorReport unwrapped, map CalciteException to SemanticCheckException, and log client errors at WARN. Also unwrap ErrorReport to its cause in the SQL error formatter so the reported type reflects the cause, not the wrapper. Signed-off-by: Chen Dai --- .../sql/api/UnifiedQueryPlanner.java | 13 +++++++++--- .../sql/api/UnifiedQueryPlannerTest.java | 18 ++++++++++++++++- .../executor/format/ErrorMessageFactory.java | 5 +++++ .../unittest/ErrorMessageFactoryTest.java | 20 +++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java index c1a9e74b763..a84300e65f8 100644 --- a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java @@ -14,6 +14,7 @@ import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.util.SqlVisitor; @@ -25,6 +26,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.exception.QueryEngineException; import org.opensearch.sql.exception.SemanticCheckException; @@ -74,13 +76,18 @@ public RelNode plan(String query) { } catch (SyntaxCheckException | QueryEngineException | UnsupportedOperationException - | IllegalArgumentException e) { - LOG.error("Failed to plan query: {}", e.getMessage()); + | IllegalArgumentException + | ErrorReport e) { + LOG.warn("Failed to plan query: {}", e.getMessage()); throw e; + } catch (CalciteException e) { + // Calcite validation errors (e.g. table not found) indicate an invalid query. + LOG.warn("Failed to plan query, invalid query: {}", e.getMessage()); + throw new SemanticCheckException(e.getMessage(), e); } catch (AssertionError e) { // Calcite throws assertion error directly when building bad RelNode String message = "Failed to plan query: invalid plan structure"; - LOG.error(message, e); + LOG.warn(message, e); throw new SemanticCheckException(message, e); } catch (Exception e) { String message = "Failed to plan query: unexpected error"; diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java index bf9ff38d694..296e9eb2519 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java @@ -10,10 +10,12 @@ import java.util.Map; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.impl.AbstractSchema; import org.junit.Test; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.executor.QueryType; @@ -72,7 +74,7 @@ public void testPPLQueryPlanningWithDefaultNamespaceMultiLevel() { // This is valid in SparkSQL, but Calcite requires "catalog" as the default root schema to // resolve it - assertThrows(IllegalStateException.class, () -> planner.plan("source = opensearch.employees")); + assertThrows(SemanticCheckException.class, () -> planner.plan("source = opensearch.employees")); } @Test @@ -131,6 +133,20 @@ public void semanticErrorIsRethrownAsSemanticCheckException() { .assertErrorMessageEquals("Source and target patterns have different wildcard counts"); } + @Test + public void fieldNotFoundIsRethrownAsErrorReport() { + givenInvalidQuery("source = catalog.employees | where unknown_field = 1") + .assertErrorType(ErrorReport.class) + .assertErrorMessageContains("Field [unknown_field] not found"); + } + + @Test + public void invalidTableIsRethrownAsSemanticCheckException() { + givenInvalidQuery("source = catalog.nonexistent_table") + .assertErrorType(SemanticCheckException.class) + .assertCauseType(CalciteException.class); + } + @Test public void assertionErrorIsWrappedAsSemanticCheckException() { // Remove when the underlying Calcite assertion is fixed. diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java index ba28ee83252..f2e0c9e1fd4 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.format; import org.opensearch.OpenSearchException; +import org.opensearch.sql.common.error.ErrorReport; public class ErrorMessageFactory { /** @@ -25,6 +26,10 @@ public static ErrorMessage createErrorMessage(Exception e, int status) { OpenSearchException exception = (OpenSearchException) unwrapCause(e); return new OpenSearchErrorMessage(exception, exception.status().getStatus()); } + // Unwrap ErrorReport so the error type reflects the underlying cause, not the wrapper. + if (e instanceof ErrorReport) { + return new ErrorMessage(((ErrorReport) e).getCause(), status); + } return new ErrorMessage(e, status); } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java index 31baaced852..0984f8e1f45 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java @@ -9,6 +9,7 @@ import org.junit.Test; import org.opensearch.OpenSearchException; import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.legacy.executor.format.ErrorMessage; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.executor.format.OpenSearchErrorMessage; @@ -42,6 +43,25 @@ public void nonOpenSearchExceptionWithWrappedEsExceptionCauseShouldCreateEsError Assert.assertTrue(msg instanceof OpenSearchErrorMessage); } + @Test + public void errorReportShouldRenderUnderlyingCauseType() { + Exception exception = + ErrorReport.wrap(new IllegalArgumentException("Field [x] not found.")).build(); + ErrorMessage msg = + ErrorMessageFactory.createErrorMessage(exception, RestStatus.BAD_REQUEST.getStatus()); + Assert.assertFalse(msg instanceof OpenSearchErrorMessage); + Assert.assertTrue(msg.toString().contains("IllegalArgumentException")); + Assert.assertFalse(msg.toString().contains("ErrorReport")); + } + + @Test + public void errorReportWrappingEsExceptionShouldCreateEsErrorMessage() { + Exception exception = ErrorReport.wrap(new OpenSearchException(nonOpenSearchThrowable)).build(); + ErrorMessage msg = + ErrorMessageFactory.createErrorMessage(exception, RestStatus.NOT_FOUND.getStatus()); + Assert.assertTrue(msg instanceof OpenSearchErrorMessage); + } + @Test public void nonOpenSearchExceptionWithMultiLayerWrappedEsExceptionCauseShouldCreateEsErrorMessage() { From 30af8b2a31f0d38ce91da06e17627f29ec8d4996 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 10 Jun 2026 15:06:44 -0700 Subject: [PATCH 26/36] [BugFix] Handle opaque NullPointerException for unresolvable alias-type field path (#5536) * Fix opaque NullPointerException for unresolvable alias-type field path When a mapping contains a field of "type": "alias" whose "path" points to a target absent from the flattened mapping (a text multi-field such as field.keyword, or a removed/renamed field), validateAliasType passed a null target into the OpenSearchAliasType constructor, which dereferenced it at super(type.getExprCoreType()) and surfaced an opaque NullPointerException. Guard the null target and throw a SemanticCheckException naming the alias field and its unresolved path. SemanticCheckException extends QueryEngineException, so JdbcResponseFormatter maps it to HTTP 400 (client error) rather than the misleading 500 a generic exception would produce. Add unit tests covering the .keyword multi-field and missing-field cases. Fixes #5535 Signed-off-by: Jialiang Liang * Add integration test and trim unit-test comments for alias path fix Add a QueryValidationIT case asserting that SELECT * over an index whose alias field targets a text multi-field (source.keyword) returns a 400 SemanticCheckException with the descriptive message. An alias pointing at a truly missing field is rejected by OpenSearch at index-creation time, so it is not reachable through the SQL plugin and is covered by the unit test only. Shorten the unit-test comments and drop inline issue references. Signed-off-by: Jialiang Liang * Apply spotless formatting Signed-off-by: Jialiang Liang * Adopt ErrorReport for unresolvable alias path error Wrap the SemanticCheckException in an ErrorReport (the report-builder interface from #5266) so the error carries structured context as it bubbles up: FIELD_NOT_FOUND code, ANALYZING stage, a location chain, the alias field and path as context, and a fix suggestion. On the PPL/Calcite path this renders as a rich structured error; on the SQL JDBC path it still returns a clear 400 (RestSqlAction unwraps to the SemanticCheckException cause), though the JdbcResponseFormatter does not yet render the ErrorReport structure. Update the unit test to assert the ErrorReport code/stage/context/cause, the SQL IT for the ErrorReport type, and add a PPL IT asserting the structured FIELD_NOT_FOUND error in CalciteErrorReportStageIT. Signed-off-by: Jialiang Liang --------- Signed-off-by: Jialiang Liang --- .../remote/CalciteErrorReportStageIT.java | 33 ++++++++++++ .../opensearch/sql/sql/QueryValidationIT.java | 25 +++++++++ .../data/type/OpenSearchDataType.java | 25 ++++++++- .../data/type/OpenSearchDataTypeTest.java | 51 +++++++++++++++++++ 4 files changed, 133 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java index f51ffabdc35..d2813f6eb17 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java @@ -11,6 +11,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -214,4 +215,36 @@ public void testStageDescriptionIsUserFriendly() throws IOException { || stageDescription.toLowerCase().contains("run") || stageDescription.toLowerCase().contains("query")); } + + // An alias field whose path targets a text multi-field (e.g. "source.keyword") is not present in + // the flattened mapping. It used to surface an opaque NullPointerException; it must now report a + // structured FIELD_NOT_FOUND error with a suggestion. + @Test + public void testAliasToUnresolvablePathIncludesStructuredError() throws IOException { + String index = "test_alias_unresolved_keyword"; + Request createIndex = new Request("PUT", "/" + index); + createIndex.setJsonEntity( + "{ \"mappings\": { \"properties\": {" + + " \"source\": { \"type\": \"text\", \"fields\": { \"keyword\": { \"type\":" + + " \"keyword\" } } }," + + " \"source_alias\": { \"type\": \"alias\", \"path\": \"source.keyword\" } } } }"); + client().performRequest(createIndex); + + ResponseException exception = + assertThrows(ResponseException.class, () -> executeQuery("source=" + index)); + + JSONObject error = + new JSONObject(getResponseBody(exception.getResponse())).getJSONObject("error"); + + assertEquals("FIELD_NOT_FOUND", error.getString("code")); + assertTrue( + "Details should name the alias field and path", + error + .getString("details") + .contains("Alias field [source_alias] refers to unresolved path [source.keyword]")); + JSONObject context = error.getJSONObject("context"); + assertEquals("source_alias", context.getString("alias_field")); + assertEquals("source.keyword", context.getString("alias_path")); + assertTrue("Should include a suggestion", error.has("suggestion")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 3373b46303c..71f16f30e43 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -103,6 +103,31 @@ private static void whenExecuteMalformedPayload() throws IOException { client().performRequest(request); } + // An alias field whose path targets a text multi-field (e.g. "source.keyword") must fail with a + // descriptive client error rather than an opaque NullPointerException. + @Test + public void testAliasToKeywordMultiFieldFailsWithBadRequest() throws IOException { + String index = "test_alias_unresolved_keyword"; + createIndexWithMapping( + index, + "{ \"properties\": {" + + " \"source\": { \"type\": \"text\", \"fields\": { \"keyword\": { \"type\":" + + " \"keyword\" } } }," + + " \"source_alias\": { \"type\": \"alias\", \"path\": \"source.keyword\" } } }"); + + expectResponseException() + .hasStatusCode(BAD_REQUEST) + .hasErrorType("ErrorReport") + .containsMessage("Alias field [source_alias] refers to unresolved path [source.keyword]") + .whenExecute(String.format(Locale.ROOT, "SELECT * FROM %s", index)); + } + + private static void createIndexWithMapping(String indexName, String mapping) throws IOException { + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity(String.format(Locale.ROOT, "{ \"mappings\": %s }", mapping)); + client().performRequest(request); + } + public ResponseExceptionAssertion expectResponseException() { return new ResponseExceptionAssertion(exceptionRule); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 79d49a143de..76de0c30a08 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -14,8 +14,12 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import org.apache.commons.lang3.EnumUtils; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; /** The extension of ExprType in OpenSearch. */ @EqualsAndHashCode @@ -297,7 +301,26 @@ private static void validateAliasType(Map result) { (key, value) -> { if (value instanceof OpenSearchAliasType && value.getOriginalPath().isPresent()) { String originalPath = value.getOriginalPath().get(); - result.put(key, new OpenSearchAliasType(originalPath, result.get(originalPath))); + OpenSearchDataType target = result.get(originalPath); + if (target == null) { + throw ErrorReport.wrap( + new SemanticCheckException( + String.format( + "Alias field [%s] refers to unresolved path [%s].", + key, originalPath))) + .code(ErrorCode.FIELD_NOT_FOUND) + .stage(QueryProcessingStage.ANALYZING) + .location("while resolving alias fields in the index mapping") + .context("alias_field", key) + .context("alias_path", originalPath) + .suggestion( + "The alias path must point to an existing field in the mapping; a text" + + " multi-field (e.g. \"" + + originalPath + + ".keyword\") or a removed/renamed field is not a valid alias target.") + .build(); + } + result.put(key, new OpenSearchAliasType(originalPath, target)); } }); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 40985130c52..1479ccfb615 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -41,8 +41,12 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchDataTypeTest { @@ -483,6 +487,53 @@ public void test_AliasType() { () -> assertEquals("original_path2", aliasTypeOnDouble.getOriginalPath().orElseThrow())); } + @Test + public void traverseAndFlatten_alias_to_unresolvable_path_throws_descriptive_error() { + // Alias path targets a text multi-field, which is not in the flattened mapping. + Map keywordAliasTree = + Map.of( + "source", + textKeywordType, + "source_alias", + new OpenSearchAliasType("source.keyword", OpenSearchDataType.of(MappingType.Invalid))); + ErrorReport keywordError = + assertThrows( + ErrorReport.class, () -> OpenSearchDataType.traverseAndFlatten(keywordAliasTree)); + assertAll( + () -> assertEquals(ErrorCode.FIELD_NOT_FOUND, keywordError.getCode()), + () -> assertEquals(QueryProcessingStage.ANALYZING, keywordError.getStage()), + () -> assertTrue(keywordError.getCause() instanceof SemanticCheckException), + () -> + assertEquals( + "Alias field [source_alias] refers to unresolved path [source.keyword].", + keywordError.getCause().getMessage()), + () -> assertEquals("source_alias", keywordError.getContext().get("alias_field")), + () -> assertEquals("source.keyword", keywordError.getContext().get("alias_path")), + () -> + assertTrue( + keywordError.getSuggestion().contains("\"source.keyword.keyword\"") + && keywordError.getSuggestion().contains("not a valid alias target"))); + + // Alias path targets a field that does not exist. + Map missingFieldTree = + Map.of( + "col1", + textType, + "col_alias", + new OpenSearchAliasType("missing", OpenSearchDataType.of(MappingType.Invalid))); + ErrorReport missingError = + assertThrows( + ErrorReport.class, () -> OpenSearchDataType.traverseAndFlatten(missingFieldTree)); + assertAll( + () -> assertEquals(ErrorCode.FIELD_NOT_FOUND, missingError.getCode()), + () -> assertTrue(missingError.getCause() instanceof SemanticCheckException), + () -> + assertEquals( + "Alias field [col_alias] refers to unresolved path [missing].", + missingError.getCause().getMessage()), + () -> assertEquals("missing", missingError.getContext().get("alias_path"))); + } + @Test public void test_parseMapping_on_AliasType() { Map indexMapping1 = From 6199484ed68a80e494f6e1f17079de6be1bcbd99 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 10 Jun 2026 16:02:35 -0700 Subject: [PATCH 27/36] case test patches (#5531) Signed-off-by: Simeon Widdis --- .../calcite/remote/CalcitePPLCaseFunctionIT.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index b7e16d1da8b..84159c1bb96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -25,6 +25,7 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; public class CalcitePPLCaseFunctionIT extends PPLIntegTestCase { + @Override public void init() throws Exception { super.init(); @@ -48,14 +49,15 @@ private void appendDataForBadResponse() throws IOException { new Request("PUT", "/" + TestsConstants.TEST_INDEX_WEBLOGS + "/_doc/8?refresh=true"); request2.setJsonEntity( "{\"host\": \"0.0.0.2\", \"method\": \"GET\", \"url\":" - + " \"/shuttle/missions/sts-73/mission-sts-73.html\", \"response\": \"500\", \"bytes\":" - + " \"4085\"}"); + + " \"/shuttle/missions/sts-73/mission-sts-73.html\", \"response\": \"500\"," + + " \"bytes\": \"4085\"}"); client().performRequest(request2); Request request3 = new Request("PUT", "/" + TestsConstants.TEST_INDEX_WEBLOGS + "/_doc/9?refresh=true"); request3.setJsonEntity( - "{\"host\": \"::3\", \"method\": \"GET\", \"url\": \"/shuttle/countdown/countdown.html\"," - + " \"response\": \"403\", \"bytes\": \"3985\"}"); + "{\"host\": \"::3\", \"method\": \"GET\", \"url\":" + + " \"/shuttle/countdown/countdown.html\", \"response\": \"403\", \"bytes\":" + + " \"3985\"}"); client().performRequest(request3); Request request4 = new Request("PUT", "/" + TestsConstants.TEST_INDEX_WEBLOGS + "/_doc/10?refresh=true"); @@ -308,7 +310,7 @@ public void testCaseCanBePushedDownAsRangeQuery() throws IOException { TEST_INDEX_BANK)); verifySchema(actual4, schema("avg(balance)", "double"), schema("age_range", "string")); // There's such a discrepancy because null cannot be the key for a range query - if (isPushdownDisabled()) { + if (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) { verifyDataRows( actual4, rows(32838.0, "u30"), @@ -463,7 +465,7 @@ public void testCaseAggWithNullValues() throws IOException { TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("avg(age)", "double"), schema("age_category", "string")); // There is such discrepancy because range aggregations will ignore null values - if (isPushdownDisabled()) { + if (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) { verifyDataRows( actual, rows(10, "teenager"), From 2850d6e3b21dd3523236d6f42eeb63d83838ce37 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Wed, 10 Jun 2026 16:38:18 -0700 Subject: [PATCH 28/36] Increment version to 3.8.0-SNAPSHOT (#5498) Signed-off-by: Eric Wei Co-authored-by: Simeon Widdis --- build.gradle | 2 +- core/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index d892c31e1c4..7b76532aad0 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.7.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.8.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/build.gradle b/core/build.gradle index 4c7cfb34235..4d0d98edb29 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -65,11 +65,11 @@ dependencies { } api 'org.apache.calcite:calcite-linq4j:1.41.0' api project(':common') - compileOnly 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT' + compileOnly 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' // Needed because analytics-api's QueryPlanExecutor signature uses // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type. compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" - testImplementation 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT' + testImplementation 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" implementation "com.github.seancfoley:ipaddress:5.4.2" implementation "com.jayway.jsonpath:json-path:2.9.0" From 7a3866430dd641dd79519aa9e958ac75b99ac9fc Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Wed, 10 Jun 2026 17:10:44 -0700 Subject: [PATCH 29/36] Fix doctest job-scheduler dependency resolution for 3.8.0 (#5540) Signed-off-by: Simeon Widdis --- doctest/build.gradle | 81 ++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 60 deletions(-) diff --git a/doctest/build.gradle b/doctest/build.gradle index d758a6de4b8..cce64170f56 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -15,6 +15,20 @@ plugins { apply plugin: 'opensearch.testclusters' +configurations { + zipArchive +} + +repositories { + mavenLocal() + maven { url "https://ci.opensearch.org/maven2/" } + mavenCentral() +} + +dependencies { + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" +} + def path = project(':').projectDir // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from // plugin module, and will only build ppl in it. @@ -156,19 +170,13 @@ check.dependsOn doctest clean.dependsOn(cleanBootstrap) clean.dependsOn(stopPrometheus) -// 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT -String opensearch_no_snapshot = opensearch_version.replace('-SNAPSHOT', '') -String[] version_tokens = opensearch_no_snapshot.tokenize('-') -String opensearch_build = version_tokens[0] + '.0' -if (version_tokens.length > 1) { - opensearch_build += '-' + version_tokens[1] +def getJobSchedulerPlugin() { + provider { (RegularFile) (() -> + configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile ) + } } -String mlCommonsRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-' + opensearch_build + '.zip' -String mlCommonsPlugin = 'opensearch-ml' - -String bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/' + - 'opensearch/plugins/opensearch-job-scheduler-' + opensearch_build + '.zip' -String jsPlugin = 'opensearch-job-scheduler' testClusters { docTestCluster { @@ -194,7 +202,7 @@ testClusters { } })) */ - plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload)) + plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' } @@ -203,50 +211,3 @@ tasks.register("runRestTestCluster", RunTask) { description = 'Runs OpenSearch SQL plugin' useCluster testClusters.docTestCluster; } - - -def getJobSchedulerPlugin(String jsPlugin, String bwcOpenSearchJSDownload) { - return provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - // Use absolute paths - String basePath = new File('.').getCanonicalPath() - File dir = new File(basePath + File.separator + 'doctest' + File.separator + jsPlugin) - - // Log the directory path for debugging - println("Creating directory: " + dir.getAbsolutePath()) - - // Create directory if it doesn't exist - if (!dir.exists()) { - if (!dir.mkdirs()) { - throw new IOException("Failed to create directory: " + dir.getAbsolutePath()) - } - } - - // Define the file path - File f = new File(dir, jsPlugin + '-' + opensearch_build + '.zip') - - // Download file if it doesn't exist - if (!f.exists()) { - println("Downloading file from: " + bwcOpenSearchJSDownload) - println("Saving to file: " + f.getAbsolutePath()) - - new URL(bwcOpenSearchJSDownload).withInputStream { ins -> - f.withOutputStream { it << ins } - } - } - - // Check if the file was created successfully - if (!f.exists()) { - throw new FileNotFoundException("File was not created: " + f.getAbsolutePath()) - } - - return fileTree(f.getParent()).matching { include f.getName() }.singleFile - } - } - } - }) -} From f7fb3d0ea6d2fd25c5d7f4277db804700b5f3341 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Wed, 10 Jun 2026 09:24:47 -0700 Subject: [PATCH 30/36] [analytics-engine] Add _ae dataset variants for analytics-engine IT coverage The analytics-engine cannot read nested/geo_point/geo_shape fields, so a handful of test datasets need an _ae sibling that drops those fields from the mapping and the matching keys from each bulk doc. Add TestUtils.AnalyticsIndexConfig#resolveDatasetPath to transparently route a dataset/mapping path to its _ae variant when the config is enabled and the sibling exists on disk; normal CI is byte-for-byte identical otherwise. Ships the _ae variants for datatypes, deep_nested, merge_test_1/2, and nested_simple, plus AnalyticsIndexConfigVariantTests covering the resolution logic. Signed-off-by: Eric Wei --- .../AnalyticsIndexConfigVariantTests.java | 81 +++++++++++++++++++ .../org/opensearch/sql/legacy/TestUtils.java | 36 ++++++++- .../README_analytics_engine_variants.md | 43 ++++++++++ .../src/test/resources/datatypes_ae.json | 2 + .../resources/deep_nested_index_data_ae.json | 2 + .../datatypes_index_mapping_ae.json | 38 +++++++++ .../deep_nested_index_mapping_ae.json | 27 +++++++ .../merge_test_1_mapping_ae.json | 30 +++++++ .../merge_test_2_mapping_ae.json | 30 +++++++ .../nested_simple_index_mapping_ae.json | 21 +++++ .../src/test/resources/merge_test_1_ae.json | 2 + .../src/test/resources/merge_test_2_ae.json | 2 + .../src/test/resources/nested_simple_ae.json | 10 +++ 13 files changed, 322 insertions(+), 2 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java create mode 100644 integ-test/src/test/resources/README_analytics_engine_variants.md create mode 100644 integ-test/src/test/resources/datatypes_ae.json create mode 100644 integ-test/src/test/resources/deep_nested_index_data_ae.json create mode 100644 integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json create mode 100644 integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json create mode 100644 integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json create mode 100644 integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json create mode 100644 integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json create mode 100644 integ-test/src/test/resources/merge_test_1_ae.json create mode 100644 integ-test/src/test/resources/merge_test_2_ae.json create mode 100644 integ-test/src/test/resources/nested_simple_ae.json diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java new file mode 100644 index 00000000000..65969ac0b95 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy; + +import static org.junit.Assert.assertEquals; + +import org.junit.After; +import org.junit.Test; +import org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig; + +/** + * Unit coverage for {@link AnalyticsIndexConfig#resolveDatasetPath(String)} — the analytics-engine + * {@code _ae} dataset-variant swap. Guards the two behaviors the feature depends on: zero change + * when the flag is off, and {@code _ae} pickup only when a sibling actually exists on disk. + */ +public class AnalyticsIndexConfigVariantTests { + + @After + public void clearFlag() { + System.clearProperty(AnalyticsIndexConfig.ENABLED_PROP); + } + + @Test + public void flagOff_returnsOriginalUnchanged() { + System.clearProperty(AnalyticsIndexConfig.ENABLED_PROP); + String mapping = "src/test/resources/indexDefinitions/datatypes_index_mapping.json"; + String data = "src/test/resources/datatypes.json"; + assertEquals(mapping, AnalyticsIndexConfig.resolveDatasetPath(mapping)); + assertEquals(data, AnalyticsIndexConfig.resolveDatasetPath(data)); + } + + @Test + public void flagOn_resolvesAeVariantWhenSiblingExists() { + System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); + // datasets that ship an _ae sibling + assertEquals( + "src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json", + AnalyticsIndexConfig.resolveDatasetPath( + "src/test/resources/indexDefinitions/datatypes_index_mapping.json")); + assertEquals( + "src/test/resources/datatypes_ae.json", + AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/datatypes.json")); + assertEquals( + "src/test/resources/nested_simple_ae.json", + AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/nested_simple.json")); + assertEquals( + "src/test/resources/deep_nested_index_data_ae.json", + AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/deep_nested_index_data.json")); + assertEquals( + "src/test/resources/merge_test_1_ae.json", + AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/merge_test_1.json")); + assertEquals( + "src/test/resources/merge_test_2_ae.json", + AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/merge_test_2.json")); + } + + @Test + public void flagOn_fallsBackWhenNoSibling() { + System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); + // no _ae sibling on disk -> original returned + String noVariant = "src/test/resources/accounts.json"; + assertEquals(noVariant, AnalyticsIndexConfig.resolveDatasetPath(noVariant)); + String cascaded = "src/test/resources/indexDefinitions/cascaded_nested_index_mapping.json"; + assertEquals(cascaded, AnalyticsIndexConfig.resolveDatasetPath(cascaded)); + } + + @Test + public void flagOn_pathWithoutExtensionReturnedAsIs() { + System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); + assertEquals("src/test/resources/noext", AnalyticsIndexConfig.resolveDatasetPath("noext")); + } + + @Test + public void nullPathReturnsNull() { + System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); + assertEquals(null, AnalyticsIndexConfig.resolveDatasetPath(null)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index f54f1c684fe..a56a9ab5b8c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -143,6 +143,36 @@ static String bulkLoadRefreshParam() { return isEnabled() ? "refresh=true" : "refresh=wait_for&wait_for_active_shards=all"; } + /** + * Resolve a test-resource path to its analytics-engine variant when one exists. The + * analytics-engine cannot read {@code nested}/{@code geo_point}/{@code geo_shape} fields, so a + * handful of test datasets ship an {@code _ae} sibling (e.g. {@code + * datatypes_index_mapping.json} → {@code datatypes_index_mapping_ae.json}, {@code + * datatypes.json} → {@code datatypes_ae.json}) that drops those fields from the mapping and the + * matching keys from each bulk doc. + * + *

    When the config is disabled, or no {@code _ae} sibling exists on disk, the original {@code + * relPath} is returned unchanged — so normal CI is byte-for-byte identical and a dataset only + * needs a variant if its fields are actually unsupported. The variant name is formed by + * inserting {@code _ae} before the final extension; paths without an extension are returned + * as-is. + * + * @param relPath project-root-relative resource path (mapping or bulk-data file) + * @return the {@code _ae} variant's relative path when enabled and present, else {@code + * relPath} + */ + static String resolveDatasetPath(String relPath) { + if (!isEnabled() || relPath == null) { + return relPath; + } + int dot = relPath.lastIndexOf('.'); + if (dot < 0) { + return relPath; + } + String variant = relPath.substring(0, dot) + "_ae" + relPath.substring(dot); + return new File(getResourceFilePath(variant)).exists() ? variant : relPath; + } + private AnalyticsIndexConfig() {} } @@ -225,7 +255,8 @@ public static boolean isIndexExist(RestClient client, String indexName) { */ public static void loadDataByRestClient( RestClient client, String indexName, String dataSetFilePath) throws IOException { - Path path = Paths.get(getResourceFilePath(dataSetFilePath)); + Path path = + Paths.get(getResourceFilePath(AnalyticsIndexConfig.resolveDatasetPath(dataSetFilePath))); Request request = new Request( "POST", "/" + indexName + "/_bulk?" + AnalyticsIndexConfig.bulkLoadRefreshParam()); @@ -663,7 +694,8 @@ public static List> getPermutations(final List items) { public static String getMappingFile(String fileName) { try { - return fileToString(MAPPING_FILE_PATH + fileName, false); + return fileToString( + AnalyticsIndexConfig.resolveDatasetPath(MAPPING_FILE_PATH + fileName), false); } catch (IOException e) { return null; } diff --git a/integ-test/src/test/resources/README_analytics_engine_variants.md b/integ-test/src/test/resources/README_analytics_engine_variants.md new file mode 100644 index 00000000000..1d8d4c5d412 --- /dev/null +++ b/integ-test/src/test/resources/README_analytics_engine_variants.md @@ -0,0 +1,43 @@ +# Analytics-engine (`_ae`) dataset variants + +The analytics-engine backend (DataFusion, reached when an index is parquet-backed — +see `RestUnifiedQueryAction.isAnalyticsIndex`) cannot read `nested`, `geo_point`, or +`geo_shape` fields. ITs run against the analytics engine by setting +`-Dtests.analytics.parquet_indices=true`, which makes every test-created index +parquet-backed. + +When that flag is on, `TestUtils.AnalyticsIndexConfig.resolveDatasetPath(...)` +transparently swaps a resource path `foo.json` → `foo_ae.json` **iff** the `_ae` +sibling exists on disk. With the flag off (normal CI), the swap never happens and the +original files are used byte-for-byte — so a dataset only needs an `_ae` variant if +its fields are actually unsupported, and adding one is a pure opt-in. + +Each `_ae` variant **drops** every `nested`/`geo_point`/`geo_shape` field from the +mapping and the matching keys from each bulk doc. (Policy: drop the field entirely — +no `nested → object` demotion.) + +## Variants present (a useful, non-empty subset survives the drop) + +| Source dataset | Dropped field(s) | Surviving columns | +| --- | --- | --- | +| `datatypes` (DATA_TYPE_NONNUMERIC) | `nested_value` (nested), `geo_point_value` (geo_point) | boolean/keyword/text/binary/date/date_nanos/ip/object | +| `nested_simple` (NESTED_SIMPLE) | `address` (nested) | `name`, `age`, `id` | +| `deep_nested` (DEEP_NESTED) | `projects` (nested); `accounts` data left out (not in explicit mapping) | `city`, `account` objects | +| `merge_test_1` (MERGE_TEST_1) | `machine_array` (nested) | `machine`, `machine_deep` objects | +| `merge_test_2` (MERGE_TEST_2) | `machine_array` (nested) | `machine`, `machine_deep` objects | + +## Deliberately NOT given a variant (assume-skip under AE instead) + +These datasets are *entirely* nested-centric — dropping the unsupported field leaves an +empty mapping/data, so a variant would restore no coverage. The tests that load them are +nested/mvexpand tests that cannot pass under AE regardless; they should assume-skip when +`isAnalyticsParquetIndicesEnabled()` is true rather than load a hollow index. + +- `cascaded_nested` (CASCADED_NESTED) — the whole mapping is one `author` nested tree. +- `mvexpand_edge_cases` (MVEXPAND_EDGE_CASES) — `skills` nested is the mvexpand target. + +## Adding a new variant + +1. Copy the mapping to `indexDefinitions/_ae.json`, remove unsupported fields. +2. Copy the bulk data to `_ae.json`, remove the matching keys from each doc. +3. Nothing else — `resolveDatasetPath` finds it automatically when AE is enabled. diff --git a/integ-test/src/test/resources/datatypes_ae.json b/integ-test/src/test/resources/datatypes_ae.json new file mode 100644 index 00000000000..370e03ac4ba --- /dev/null +++ b/integ-test/src/test/resources/datatypes_ae.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}} diff --git a/integ-test/src/test/resources/deep_nested_index_data_ae.json b/integ-test/src/test/resources/deep_nested_index_data_ae.json new file mode 100644 index 00000000000..e7dda50a1f3 --- /dev/null +++ b/integ-test/src/test/resources/deep_nested_index_data_ae.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"city": {"name": "Seattle", "location": {"latitude": 10.5}}} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json new file mode 100644 index 00000000000..f41b0069896 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json @@ -0,0 +1,38 @@ +{ + "mappings": { + "properties": { + "boolean_value": { + "type": "boolean" + }, + "keyword_value": { + "type": "keyword" + }, + "text_value": { + "type": "text" + }, + "binary_value": { + "type": "binary" + }, + "date_value": { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" + }, + "date_nanos_value": { + "type" : "date_nanos" + }, + "ip_value": { + "type": "ip" + }, + "object_value": { + "properties": { + "first": { + "type": "text" + }, + "last": { + "type": "text" + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json new file mode 100644 index 00000000000..2057092296d --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json @@ -0,0 +1,27 @@ +{ + "mappings": { + "properties": { + "city": { + "properties": { + "name": { + "type": "keyword" + }, + "location": { + "properties": { + "latitude": { + "type": "double" + } + } + } + } + }, + "account": { + "properties": { + "id": { + "type": "keyword" + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json new file mode 100644 index 00000000000..1d8c5f555fe --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json @@ -0,0 +1,30 @@ +{ + "mappings": { + "properties": { + "machine": { + "properties": { + "os1": { + "type": "text" + }, + "ram1": { + "type": "long" + } + } + }, + "machine_deep": { + "properties": { + "attr1": { + "type": "long" + }, + "layer": { + "properties": { + "os1": { + "type": "text" + } + } + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json new file mode 100644 index 00000000000..61b332c0332 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json @@ -0,0 +1,30 @@ +{ + "mappings": { + "properties": { + "machine": { + "properties": { + "os2": { + "type": "text" + }, + "ram2": { + "type": "long" + } + } + }, + "machine_deep": { + "properties": { + "attr2": { + "type": "long" + }, + "layer": { + "properties": { + "os2": { + "type": "text" + } + } + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json new file mode 100644 index 00000000000..bcecb2d922b --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json @@ -0,0 +1,21 @@ +{ + "mappings": { + "properties": { + "age": { + "type": "long" + }, + "id": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/merge_test_1_ae.json b/integ-test/src/test/resources/merge_test_1_ae.json new file mode 100644 index 00000000000..a852c63dac4 --- /dev/null +++ b/integ-test/src/test/resources/merge_test_1_ae.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"machine": {"os1": "linux", "ram1": 120}, "machine_deep": {"attr1": 1, "layer": {"os1": "os1"}}} diff --git a/integ-test/src/test/resources/merge_test_2_ae.json b/integ-test/src/test/resources/merge_test_2_ae.json new file mode 100644 index 00000000000..9f3193ceb6e --- /dev/null +++ b/integ-test/src/test/resources/merge_test_2_ae.json @@ -0,0 +1,2 @@ +{"index": {}} +{"machine": {"os2": "linux", "ram2": 120}, "machine_deep": {"attr2": 2, "layer": {"os2": "os2"}}} diff --git a/integ-test/src/test/resources/nested_simple_ae.json b/integ-test/src/test/resources/nested_simple_ae.json new file mode 100644 index 00000000000..67363af31c6 --- /dev/null +++ b/integ-test/src/test/resources/nested_simple_ae.json @@ -0,0 +1,10 @@ +{"index":{"_id":"1"}} +{"name":"abbas","age":24} +{"index":{"_id":"2"}} +{"name":"chen","age":32} +{"index":{"_id":"3"}} +{"name":"peng","age":26} +{"index":{"_id":"4"}} +{"name":"andy","age":19,"id":4} +{"index":{"_id":"5"}} +{"name":"david","age":25} From c34b869e58b21f13529f9749bbff5dd0486a0a72 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Wed, 10 Jun 2026 12:04:53 -0700 Subject: [PATCH 31/36] [analytics-engine] Strip AE-unsupported fields from test datasets at load The analytics-engine (DataFusion) backend cannot read nested, geo_point, geo_shape, binary, or alias fields. Rather than maintain per-index _ae dataset variants, strip those fields uniformly at load time on the AE route: - createIndexByRestClient recursively removes unsupported-typed properties from the mapping (including object sub-properties) and reports the top-level fields dropped. - loadIndex passes that set to a new loadDataByRestClient overload, which strips the matching keys from each bulk source doc so mapping and data agree. Gated on tests.analytics.parquet_indices; byte-for-byte no-op off the AE route. Replaces the earlier filename-_ae hook + hand-authored variant files. Signed-off-by: Eric Wei --- .../sql/legacy/AnalyticsFieldStripTests.java | 115 ++++++++++++++ .../AnalyticsIndexConfigVariantTests.java | 81 ---------- .../sql/legacy/SQLIntegTestCase.java | 4 +- .../org/opensearch/sql/legacy/TestUtils.java | 149 ++++++++++++++---- .../README_analytics_engine_variants.md | 43 ----- .../src/test/resources/datatypes_ae.json | 2 - .../resources/deep_nested_index_data_ae.json | 2 - .../datatypes_index_mapping_ae.json | 38 ----- .../deep_nested_index_mapping_ae.json | 27 ---- .../merge_test_1_mapping_ae.json | 30 ---- .../merge_test_2_mapping_ae.json | 30 ---- .../nested_simple_index_mapping_ae.json | 21 --- .../src/test/resources/merge_test_1_ae.json | 2 - .../src/test/resources/merge_test_2_ae.json | 2 - .../src/test/resources/nested_simple_ae.json | 10 -- 15 files changed, 239 insertions(+), 317 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java delete mode 100644 integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java delete mode 100644 integ-test/src/test/resources/README_analytics_engine_variants.md delete mode 100644 integ-test/src/test/resources/datatypes_ae.json delete mode 100644 integ-test/src/test/resources/deep_nested_index_data_ae.json delete mode 100644 integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json delete mode 100644 integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json delete mode 100644 integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json delete mode 100644 integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json delete mode 100644 integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json delete mode 100644 integ-test/src/test/resources/merge_test_1_ae.json delete mode 100644 integ-test/src/test/resources/merge_test_2_ae.json delete mode 100644 integ-test/src/test/resources/nested_simple_ae.json diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java new file mode 100644 index 00000000000..557a79294ad --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -0,0 +1,115 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Set; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Test; +import org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig; + +/** + * Pure-logic coverage for the analytics-engine field strip (no filesystem / cluster). Verifies that + * nested/geo_point/geo_shape/binary/alias fields are removed from mappings and bulk data on the AE + * route, and that everything is a no-op when the route is off. + */ +public class AnalyticsFieldStripTests { + + @After + public void clearFlag() { + System.clearProperty(AnalyticsIndexConfig.ENABLED_PROP); + } + + private void enable() { + System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); + } + + private static final String MAPPING = + "{\"mappings\":{\"properties\":{" + + "\"keep_text\":{\"type\":\"text\"}," + + "\"nested_value\":{\"type\":\"nested\"}," + + "\"geo_point_value\":{\"type\":\"geo_point\"}," + + "\"geo_shape_value\":{\"type\":\"geo_shape\"}," + + "\"binary_value\":{\"type\":\"binary\"}," + + "\"alias_value\":{\"type\":\"alias\",\"path\":\"keep_text\"}," + + "\"obj\":{\"properties\":{" + + " \"keep_inner\":{\"type\":\"keyword\"}," + + " \"inner_geo\":{\"type\":\"geo_point\"}}}" + + "}}}"; + + @Test + public void mappingStrip_removesUnsupportedRecursively_andReportsTopLevel() { + enable(); + JSONObject json = new JSONObject(MAPPING); + Set dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + + assertEquals( + Set.of("nested_value", "geo_point_value", "geo_shape_value", "binary_value", "alias_value"), + dropped); + + JSONObject props = json.getJSONObject("mappings").getJSONObject("properties"); + assertTrue("supported scalar kept", props.has("keep_text")); + assertFalse(props.has("nested_value")); + assertFalse(props.has("geo_point_value")); + assertFalse(props.has("geo_shape_value")); + assertFalse(props.has("binary_value")); + assertFalse(props.has("alias_value")); + + // object field kept, but its unsupported sub-property stripped recursively + assertTrue(props.has("obj")); + JSONObject inner = props.getJSONObject("obj").getJSONObject("properties"); + assertTrue(inner.has("keep_inner")); + assertFalse("nested geo_point inside object dropped", inner.has("inner_geo")); + } + + @Test + public void mappingStrip_noopWhenDisabled() { + JSONObject json = new JSONObject(MAPPING); + Set dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + assertTrue(dropped.isEmpty()); + assertTrue(json.getJSONObject("mappings").getJSONObject("properties").has("nested_value")); + } + + @Test + public void bulkStrip_removesDroppedKeysFromSourceLinesOnly() { + enable(); + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"binary_value\":\"AA==\"}\n" + + "{\"index\":{\"_id\":\"2\"}}\n" + + "{\"keep_text\":\"y\",\"nested_value\":[{\"a\":1}]}\n"; + String out = + AnalyticsIndexConfig.stripBulkFields( + bulk, Set.of("geo_point_value", "binary_value", "nested_value")); + + String[] lines = out.split("\n"); + // action lines untouched + assertTrue(lines[0].contains("\"index\"")); + assertTrue(lines[2].contains("\"index\"")); + // source lines stripped, supported field retained + JSONObject doc1 = new JSONObject(lines[1]); + assertTrue(doc1.has("keep_text")); + assertFalse(doc1.has("geo_point_value")); + assertFalse(doc1.has("binary_value")); + JSONObject doc2 = new JSONObject(lines[3]); + assertTrue(doc2.has("keep_text")); + assertFalse(doc2.has("nested_value")); + } + + @Test + public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { + String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; + // disabled -> unchanged even with a drop set + assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of("geo_point_value"))); + // enabled but empty drop set -> unchanged + enable(); + assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of())); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java deleted file mode 100644 index 65969ac0b95..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsIndexConfigVariantTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.legacy; - -import static org.junit.Assert.assertEquals; - -import org.junit.After; -import org.junit.Test; -import org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig; - -/** - * Unit coverage for {@link AnalyticsIndexConfig#resolveDatasetPath(String)} — the analytics-engine - * {@code _ae} dataset-variant swap. Guards the two behaviors the feature depends on: zero change - * when the flag is off, and {@code _ae} pickup only when a sibling actually exists on disk. - */ -public class AnalyticsIndexConfigVariantTests { - - @After - public void clearFlag() { - System.clearProperty(AnalyticsIndexConfig.ENABLED_PROP); - } - - @Test - public void flagOff_returnsOriginalUnchanged() { - System.clearProperty(AnalyticsIndexConfig.ENABLED_PROP); - String mapping = "src/test/resources/indexDefinitions/datatypes_index_mapping.json"; - String data = "src/test/resources/datatypes.json"; - assertEquals(mapping, AnalyticsIndexConfig.resolveDatasetPath(mapping)); - assertEquals(data, AnalyticsIndexConfig.resolveDatasetPath(data)); - } - - @Test - public void flagOn_resolvesAeVariantWhenSiblingExists() { - System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); - // datasets that ship an _ae sibling - assertEquals( - "src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json", - AnalyticsIndexConfig.resolveDatasetPath( - "src/test/resources/indexDefinitions/datatypes_index_mapping.json")); - assertEquals( - "src/test/resources/datatypes_ae.json", - AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/datatypes.json")); - assertEquals( - "src/test/resources/nested_simple_ae.json", - AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/nested_simple.json")); - assertEquals( - "src/test/resources/deep_nested_index_data_ae.json", - AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/deep_nested_index_data.json")); - assertEquals( - "src/test/resources/merge_test_1_ae.json", - AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/merge_test_1.json")); - assertEquals( - "src/test/resources/merge_test_2_ae.json", - AnalyticsIndexConfig.resolveDatasetPath("src/test/resources/merge_test_2.json")); - } - - @Test - public void flagOn_fallsBackWhenNoSibling() { - System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); - // no _ae sibling on disk -> original returned - String noVariant = "src/test/resources/accounts.json"; - assertEquals(noVariant, AnalyticsIndexConfig.resolveDatasetPath(noVariant)); - String cascaded = "src/test/resources/indexDefinitions/cascaded_nested_index_mapping.json"; - assertEquals(cascaded, AnalyticsIndexConfig.resolveDatasetPath(cascaded)); - } - - @Test - public void flagOn_pathWithoutExtensionReturnedAsIs() { - System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); - assertEquals("src/test/resources/noext", AnalyticsIndexConfig.resolveDatasetPath("noext")); - } - - @Test - public void nullPathReturnsNull() { - System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); - assertEquals(null, AnalyticsIndexConfig.resolveDatasetPath(null)); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 183a060f7f0..67c7e3a9c74 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -214,7 +214,9 @@ protected synchronized void loadIndex(Index index, RestClient client) throws IOE if (!isIndexExist(client, indexName)) { createIndexByRestClient(client, indexName, mapping); - loadDataByRestClient(client, indexName, dataSet); + // On the analytics-engine route, unsupported-typed fields are stripped from the mapping; drop + // the same keys from the bulk data so the two agree. Empty (no-op) off the AE route. + loadDataByRestClient(client, indexName, dataSet, analyticsDroppedFields(mapping)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index a56a9ab5b8c..2510e34bbd1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -19,9 +19,11 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.stream.Collectors; import org.json.JSONArray; import org.json.JSONObject; @@ -144,33 +146,96 @@ static String bulkLoadRefreshParam() { } /** - * Resolve a test-resource path to its analytics-engine variant when one exists. The - * analytics-engine cannot read {@code nested}/{@code geo_point}/{@code geo_shape} fields, so a - * handful of test datasets ship an {@code _ae} sibling (e.g. {@code - * datatypes_index_mapping.json} → {@code datatypes_index_mapping_ae.json}, {@code - * datatypes.json} → {@code datatypes_ae.json}) that drops those fields from the mapping and the - * matching keys from each bulk doc. - * - *

    When the config is disabled, or no {@code _ae} sibling exists on disk, the original {@code - * relPath} is returned unchanged — so normal CI is byte-for-byte identical and a dataset only - * needs a variant if its fields are actually unsupported. The variant name is formed by - * inserting {@code _ae} before the final extension; paths without an extension are returned - * as-is. + * Field types the analytics-engine (DataFusion) backend cannot read. Any property of one of + * these types is removed from the mapping (and the matching key from each bulk doc) before a + * test index is created/loaded on the analytics-engine route, so seeding a dataset that happens + * to use one of them doesn't fail the whole test. Applied uniformly to every dataset — no + * per-index variant files to author or keep in sync. + */ + private static final Set UNSUPPORTED_FIELD_TYPES = + Set.of("nested", "geo_point", "geo_shape", "binary", "alias"); + + /** + * Remove every {@link #UNSUPPORTED_FIELD_TYPES} property (recursively, including object + * sub-properties) from an index-creation JSON's {@code mappings.properties}, and report the set + * of top-level field names that were dropped so the matching keys can be stripped from + * bulk data. No-op when disabled. Mutates {@code jsonObject} in place. * - * @param relPath project-root-relative resource path (mapping or bulk-data file) - * @return the {@code _ae} variant's relative path when enabled and present, else {@code - * relPath} + * @return the top-level property names removed (empty when disabled or nothing matched) */ - static String resolveDatasetPath(String relPath) { - if (!isEnabled() || relPath == null) { - return relPath; + static Set stripUnsupportedMappingFields(JSONObject jsonObject) { + if (!isEnabled() || !jsonObject.has("mappings")) { + return Set.of(); } - int dot = relPath.lastIndexOf('.'); - if (dot < 0) { - return relPath; + JSONObject mappings = jsonObject.getJSONObject("mappings"); + if (!mappings.has("properties")) { + return Set.of(); } - String variant = relPath.substring(0, dot) + "_ae" + relPath.substring(dot); - return new File(getResourceFilePath(variant)).exists() ? variant : relPath; + Set droppedTopLevel = new HashSet<>(); + JSONObject properties = mappings.getJSONObject("properties"); + for (String field : properties.keySet().toArray(new String[0])) { + if (removeIfUnsupported(properties, field)) { + droppedTopLevel.add(field); + } + } + return droppedTopLevel; + } + + /** + * If {@code properties[field]} is an unsupported type, remove it and return true. Otherwise + * recurse into its nested {@code properties} (a plain {@code object} field) and keep it. + */ + private static boolean removeIfUnsupported(JSONObject properties, String field) { + JSONObject def = properties.optJSONObject(field); + if (def == null) { + return false; + } + String type = def.optString("type", null); + if (type != null && UNSUPPORTED_FIELD_TYPES.contains(type)) { + properties.remove(field); + return true; + } + if (def.has("properties")) { + JSONObject sub = def.getJSONObject("properties"); + for (String child : sub.keySet().toArray(new String[0])) { + removeIfUnsupported(sub, child); + } + } + return false; + } + + /** + * Strip the given top-level keys from every document line of a bulk NDJSON payload. Bulk format + * alternates an action line ({@code {"index":{...}}}) with a source line; only source lines + * (those without a bulk action key) are rewritten. No-op when disabled or {@code droppedFields} + * is empty. + */ + static String stripBulkFields(String bulkBody, Set droppedFields) { + if (!isEnabled() || droppedFields.isEmpty()) { + return bulkBody; + } + String[] lines = bulkBody.split("\n", -1); + StringBuilder out = new StringBuilder(bulkBody.length()); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + String trimmed = line.trim(); + if (!trimmed.isEmpty() && trimmed.charAt(0) == '{') { + JSONObject doc = new JSONObject(trimmed); + boolean isActionLine = + doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); + if (!isActionLine) { + for (String f : droppedFields) { + doc.remove(f); + } + line = doc.toString(); + } + } + out.append(line); + if (i < lines.length - 1) { + out.append('\n'); + } + } + return out.toString(); } private AnalyticsIndexConfig() {} @@ -188,10 +253,26 @@ public static void createIndexByRestClient(RestClient client, String indexName, JSONObject jsonObject = isNullOrEmpty(mapping) ? new JSONObject() : new JSONObject(mapping); setZeroReplicas(jsonObject); AnalyticsIndexConfig.applyIndexCreationSettings(jsonObject); + // On the analytics-engine route, drop fields of types DataFusion can't read so the index can + // still be created from datasets that happen to use them. No-op otherwise. + AnalyticsIndexConfig.stripUnsupportedMappingFields(jsonObject); request.setJsonEntity(jsonObject.toString()); performRequest(client, request); } + /** + * Top-level field names the analytics-engine route would strip from {@code mapping}. Used by + * {@link #loadDataByRestClient(RestClient, String, String, java.util.Set)} so bulk docs drop the + * same keys the mapping dropped. Returns an empty set when AE is disabled or {@code mapping} is + * empty. + */ + public static Set analyticsDroppedFields(String mapping) { + if (isNullOrEmpty(mapping)) { + return Set.of(); + } + return AnalyticsIndexConfig.stripUnsupportedMappingFields(new JSONObject(mapping)); + } + /** * Sets number_of_replicas to 0 in the index settings. This makes multi-node behavior consistent * (4261) and prevents tests @@ -255,12 +336,25 @@ public static boolean isIndexExist(RestClient client, String indexName) { */ public static void loadDataByRestClient( RestClient client, String indexName, String dataSetFilePath) throws IOException { - Path path = - Paths.get(getResourceFilePath(AnalyticsIndexConfig.resolveDatasetPath(dataSetFilePath))); + loadDataByRestClient(client, indexName, dataSetFilePath, Set.of()); + } + + /** + * Same as {@link #loadDataByRestClient(RestClient, String, String)} but strips {@code + * droppedFields} (the keys removed from the mapping on the analytics-engine route) from every + * bulk source doc, so the index mapping and the data agree. When AE is disabled or {@code + * droppedFields} is empty this is byte-for-byte identical to the 3-arg form. + */ + public static void loadDataByRestClient( + RestClient client, String indexName, String dataSetFilePath, Set droppedFields) + throws IOException { + Path path = Paths.get(getResourceFilePath(dataSetFilePath)); + String body = new String(Files.readAllBytes(path)); + body = AnalyticsIndexConfig.stripBulkFields(body, droppedFields); Request request = new Request( "POST", "/" + indexName + "/_bulk?" + AnalyticsIndexConfig.bulkLoadRefreshParam()); - request.setJsonEntity(new String(Files.readAllBytes(path))); + request.setJsonEntity(body); performRequest(client, request); } @@ -694,8 +788,7 @@ public static List> getPermutations(final List items) { public static String getMappingFile(String fileName) { try { - return fileToString( - AnalyticsIndexConfig.resolveDatasetPath(MAPPING_FILE_PATH + fileName), false); + return fileToString(MAPPING_FILE_PATH + fileName, false); } catch (IOException e) { return null; } diff --git a/integ-test/src/test/resources/README_analytics_engine_variants.md b/integ-test/src/test/resources/README_analytics_engine_variants.md deleted file mode 100644 index 1d8d4c5d412..00000000000 --- a/integ-test/src/test/resources/README_analytics_engine_variants.md +++ /dev/null @@ -1,43 +0,0 @@ -# Analytics-engine (`_ae`) dataset variants - -The analytics-engine backend (DataFusion, reached when an index is parquet-backed — -see `RestUnifiedQueryAction.isAnalyticsIndex`) cannot read `nested`, `geo_point`, or -`geo_shape` fields. ITs run against the analytics engine by setting -`-Dtests.analytics.parquet_indices=true`, which makes every test-created index -parquet-backed. - -When that flag is on, `TestUtils.AnalyticsIndexConfig.resolveDatasetPath(...)` -transparently swaps a resource path `foo.json` → `foo_ae.json` **iff** the `_ae` -sibling exists on disk. With the flag off (normal CI), the swap never happens and the -original files are used byte-for-byte — so a dataset only needs an `_ae` variant if -its fields are actually unsupported, and adding one is a pure opt-in. - -Each `_ae` variant **drops** every `nested`/`geo_point`/`geo_shape` field from the -mapping and the matching keys from each bulk doc. (Policy: drop the field entirely — -no `nested → object` demotion.) - -## Variants present (a useful, non-empty subset survives the drop) - -| Source dataset | Dropped field(s) | Surviving columns | -| --- | --- | --- | -| `datatypes` (DATA_TYPE_NONNUMERIC) | `nested_value` (nested), `geo_point_value` (geo_point) | boolean/keyword/text/binary/date/date_nanos/ip/object | -| `nested_simple` (NESTED_SIMPLE) | `address` (nested) | `name`, `age`, `id` | -| `deep_nested` (DEEP_NESTED) | `projects` (nested); `accounts` data left out (not in explicit mapping) | `city`, `account` objects | -| `merge_test_1` (MERGE_TEST_1) | `machine_array` (nested) | `machine`, `machine_deep` objects | -| `merge_test_2` (MERGE_TEST_2) | `machine_array` (nested) | `machine`, `machine_deep` objects | - -## Deliberately NOT given a variant (assume-skip under AE instead) - -These datasets are *entirely* nested-centric — dropping the unsupported field leaves an -empty mapping/data, so a variant would restore no coverage. The tests that load them are -nested/mvexpand tests that cannot pass under AE regardless; they should assume-skip when -`isAnalyticsParquetIndicesEnabled()` is true rather than load a hollow index. - -- `cascaded_nested` (CASCADED_NESTED) — the whole mapping is one `author` nested tree. -- `mvexpand_edge_cases` (MVEXPAND_EDGE_CASES) — `skills` nested is the mvexpand target. - -## Adding a new variant - -1. Copy the mapping to `indexDefinitions/_ae.json`, remove unsupported fields. -2. Copy the bulk data to `_ae.json`, remove the matching keys from each doc. -3. Nothing else — `resolveDatasetPath` finds it automatically when AE is enabled. diff --git a/integ-test/src/test/resources/datatypes_ae.json b/integ-test/src/test/resources/datatypes_ae.json deleted file mode 100644 index 370e03ac4ba..00000000000 --- a/integ-test/src/test/resources/datatypes_ae.json +++ /dev/null @@ -1,2 +0,0 @@ -{"index":{"_id":"1"}} -{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}} diff --git a/integ-test/src/test/resources/deep_nested_index_data_ae.json b/integ-test/src/test/resources/deep_nested_index_data_ae.json deleted file mode 100644 index e7dda50a1f3..00000000000 --- a/integ-test/src/test/resources/deep_nested_index_data_ae.json +++ /dev/null @@ -1,2 +0,0 @@ -{"index":{"_id":"1"}} -{"city": {"name": "Seattle", "location": {"latitude": 10.5}}} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json deleted file mode 100644 index f41b0069896..00000000000 --- a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping_ae.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "mappings": { - "properties": { - "boolean_value": { - "type": "boolean" - }, - "keyword_value": { - "type": "keyword" - }, - "text_value": { - "type": "text" - }, - "binary_value": { - "type": "binary" - }, - "date_value": { - "type" : "date", - "format": "yyyy-MM-dd HH:mm:ss" - }, - "date_nanos_value": { - "type" : "date_nanos" - }, - "ip_value": { - "type": "ip" - }, - "object_value": { - "properties": { - "first": { - "type": "text" - }, - "last": { - "type": "text" - } - } - } - } - } -} diff --git a/integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json deleted file mode 100644 index 2057092296d..00000000000 --- a/integ-test/src/test/resources/indexDefinitions/deep_nested_index_mapping_ae.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "mappings": { - "properties": { - "city": { - "properties": { - "name": { - "type": "keyword" - }, - "location": { - "properties": { - "latitude": { - "type": "double" - } - } - } - } - }, - "account": { - "properties": { - "id": { - "type": "keyword" - } - } - } - } - } -} diff --git a/integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json deleted file mode 100644 index 1d8c5f555fe..00000000000 --- a/integ-test/src/test/resources/indexDefinitions/merge_test_1_mapping_ae.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "mappings": { - "properties": { - "machine": { - "properties": { - "os1": { - "type": "text" - }, - "ram1": { - "type": "long" - } - } - }, - "machine_deep": { - "properties": { - "attr1": { - "type": "long" - }, - "layer": { - "properties": { - "os1": { - "type": "text" - } - } - } - } - } - } - } -} diff --git a/integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json deleted file mode 100644 index 61b332c0332..00000000000 --- a/integ-test/src/test/resources/indexDefinitions/merge_test_2_mapping_ae.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "mappings": { - "properties": { - "machine": { - "properties": { - "os2": { - "type": "text" - }, - "ram2": { - "type": "long" - } - } - }, - "machine_deep": { - "properties": { - "attr2": { - "type": "long" - }, - "layer": { - "properties": { - "os2": { - "type": "text" - } - } - } - } - } - } - } -} diff --git a/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json deleted file mode 100644 index bcecb2d922b..00000000000 --- a/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping_ae.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "mappings": { - "properties": { - "age": { - "type": "long" - }, - "id": { - "type": "long" - }, - "name": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } -} diff --git a/integ-test/src/test/resources/merge_test_1_ae.json b/integ-test/src/test/resources/merge_test_1_ae.json deleted file mode 100644 index a852c63dac4..00000000000 --- a/integ-test/src/test/resources/merge_test_1_ae.json +++ /dev/null @@ -1,2 +0,0 @@ -{"index":{"_id":"1"}} -{"machine": {"os1": "linux", "ram1": 120}, "machine_deep": {"attr1": 1, "layer": {"os1": "os1"}}} diff --git a/integ-test/src/test/resources/merge_test_2_ae.json b/integ-test/src/test/resources/merge_test_2_ae.json deleted file mode 100644 index 9f3193ceb6e..00000000000 --- a/integ-test/src/test/resources/merge_test_2_ae.json +++ /dev/null @@ -1,2 +0,0 @@ -{"index": {}} -{"machine": {"os2": "linux", "ram2": 120}, "machine_deep": {"attr2": 2, "layer": {"os2": "os2"}}} diff --git a/integ-test/src/test/resources/nested_simple_ae.json b/integ-test/src/test/resources/nested_simple_ae.json deleted file mode 100644 index 67363af31c6..00000000000 --- a/integ-test/src/test/resources/nested_simple_ae.json +++ /dev/null @@ -1,10 +0,0 @@ -{"index":{"_id":"1"}} -{"name":"abbas","age":24} -{"index":{"_id":"2"}} -{"name":"chen","age":32} -{"index":{"_id":"3"}} -{"name":"peng","age":26} -{"index":{"_id":"4"}} -{"name":"andy","age":19,"id":4} -{"index":{"_id":"5"}} -{"name":"david","age":25} From eb127603d3ebeeeec0787c96fcdbd238d74bf59a Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Wed, 10 Jun 2026 16:25:16 -0700 Subject: [PATCH 32/36] [analytics-engine] Verify + complete unsupported-field strip on AE route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an exhaustive, fail-loud verifier IT (AnalyticsUnsupportedFieldStripVerifyIT) that loads every Index enum dataset through the real loadIndex harness on the AE route and asserts no nested/geo_point/geo_shape/binary/alias field survives in any live mapping — converting the "silent expected:<1> but was:<0>" risk into a direct, attributable failure. Skip CalciteAliasFieldAggregationIT on the AE route: it creates its index via a raw inline PUT (bypassing the load-path strip) and its queries reference alias fields directly, so alias aggregation is inherently AE-incompatible — there is nothing to salvage by stripping. Matches the existing Assume.assumeFalse idiom. Full calcite/remote PPL sweep on the AE route now has zero ingestion failures attributable to the five unsupported field types. Signed-off-by: Eric Wei --- ...nalyticsUnsupportedFieldStripVerifyIT.java | 185 ++++++++++++++++++ .../CalciteAliasFieldAggregationIT.java | 7 + 2 files changed, 192 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java new file mode 100644 index 00000000000..3a7a6feaa66 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java @@ -0,0 +1,185 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.json.JSONObject; +import org.junit.Assume; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Exhaustive, fail-loud proof that data ingestion on the analytics-engine route is free of + * unsupported-field problems. For EVERY {@link Index} value, it runs the real {@link #loadIndex} + * harness path against the AE cluster, then verifies three independent legs: + * + *

      + *
    1. Create succeeded — the index exists (a mapping that still contained + * nested/geo_point/geo_shape/binary/alias would have been rejected by the parquet/composite + * store at PUT time, so existence proves the strip removed them). + *
    2. Mapping is clean — the live mapping pulled back from the cluster contains none of + * the unsupported types at any depth. + *
    3. Data agrees with mapping — every doc is searchable and no doc carries a stripped key + * (a leftover key would dynamic-map the field back, re-introducing the unsupported type — we + * assert the live mapping stays clean after refresh and that doc count > 0 where data + * exists). + *
    + * + *

    This converts the "silent failure" risk (a missed strip surfacing only as an unrelated {@code + * expected:<1> but was:<0>} assertion in some downstream IT) into a direct, attributable failure on + * the exact index and field. Only runs on the AE route ({@code -Dtests.analytics.parquet_indices}); + * assume-skipped otherwise. + */ +public class AnalyticsUnsupportedFieldStripVerifyIT extends PPLIntegTestCase { + + private static final Set UNSUPPORTED = + Set.of("nested", "geo_point", "geo_shape", "binary", "alias"); + + /** + * Field types the parquet/composite store also rejects but that are out of scope for the strip + * (by product decision). An index whose creation fails solely because of one of these is skipped, + * not reported — this proof is strictly about {@link #UNSUPPORTED}. + */ + private static final Set OUT_OF_SCOPE_TYPES = Set.of("join"); + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { + Assume.assumeTrue( + "AE-route only: requires -Dtests.analytics.parquet_indices=true", + isAnalyticsParquetIndicesEnabled()); + + List failures = new ArrayList<>(); + for (Index index : Index.values()) { + String name = index.getName(); + try { + loadIndex(index); + } catch (Exception e) { + // A genuinely-absent dataset file is a pre-existing repo issue (dangling enum entry), not + // an + // unsupported-field ingestion problem — skip it so this stays a clean, attributable proof. + if (isMissingDatasetFile(e)) { + continue; + } + // An index that fails creation solely because of an out-of-scope type (e.g. join) is not + // part of the fix we're verifying — skip rather than report. + if (failsOnlyOnOutOfScopeType(e)) { + continue; + } + failures.add( + "[" + + index.name() + + " -> " + + name + + "] loadIndex FAILED (ingestion error): " + + rootMessage(e)); + continue; + } + // Leg 1+2: index exists and its live mapping carries no unsupported type at any depth. + List offending = unsupportedFieldsInLiveMapping(name); + if (!offending.isEmpty()) { + failures.add( + "[" + + index.name() + + " -> " + + name + + "] live mapping still has unsupported fields: " + + offending); + } + } + + if (!failures.isEmpty()) { + throw new AssertionError( + "Unsupported-field ingestion problems on the AE route (" + + failures.size() + + "):\n " + + String.join("\n ", failures)); + } + } + + /** Pull the live mapping back from the cluster and collect any unsupported-typed field paths. */ + private List unsupportedFieldsInLiveMapping(String indexName) throws IOException { + Request request = new Request("GET", "/" + indexName + "/_mapping"); + Response response = client().performRequest(request); + JSONObject body = new JSONObject(getResponseBody(response)); + // shape: { indexName: { mappings: { properties: {...} } } } + JSONObject mappings = body.getJSONObject(indexName).getJSONObject("mappings"); + List offending = new ArrayList<>(); + if (mappings.has("properties")) { + collectUnsupported(mappings.getJSONObject("properties"), "", offending); + } + return offending; + } + + private void collectUnsupported(JSONObject properties, String prefix, List out) { + for (String field : properties.keySet()) { + JSONObject def = properties.optJSONObject(field); + if (def == null) { + continue; + } + String type = def.optString("type", null); + String path = prefix.isEmpty() ? field : prefix + "." + field; + if (type != null && UNSUPPORTED.contains(type)) { + out.add(path + ":" + type); + } + if (def.has("properties")) { + collectUnsupported(def.getJSONObject("properties"), path, out); + } + } + } + + /** + * True when the index-creation error names an out-of-scope field type (e.g. the parquet store's + * "... not supported for field: X of type: join" message). Such an index is not something this + * proof is responsible for; skip it. + */ + private static boolean failsOnlyOnOutOfScopeType(Throwable t) { + String msg = rootMessage(t); + if (msg == null) { + return false; + } + for (String type : OUT_OF_SCOPE_TYPES) { + if (msg.contains("type: " + type) || msg.contains("type [" + type + "]")) { + return true; + } + } + return false; + } + + /** True when the failure is a missing dataset file on disk (pre-existing dangling enum entry). */ + private static boolean isMissingDatasetFile(Throwable t) { + for (Throwable r = t; r != null && r.getCause() != r; r = r.getCause()) { + if (r instanceof java.nio.file.NoSuchFileException + || r instanceof java.io.FileNotFoundException) { + return true; + } + if (r.getCause() == null) { + break; + } + } + return false; + } + + private static String rootMessage(Throwable t) { + Throwable r = t; + while (r.getCause() != null && r.getCause() != r) { + r = r.getCause(); + } + return r.getMessage(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java index dfe6ba17b62..6fb5cfcd0ee 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.List; import org.json.JSONObject; +import org.junit.Assume; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.client.ResponseException; @@ -31,6 +32,12 @@ public class CalciteAliasFieldAggregationIT extends PPLIntegTestCase { @Override public void init() throws Exception { super.init(); + // Alias fields are unsupported on the analytics-engine (parquet/composite) route — the index + // can't even be created, and these tests query the alias fields directly. There's nothing to + // salvage by stripping, so skip the whole class on the AE route. Runs normally otherwise. + Assume.assumeFalse( + "Alias-field aggregation is unsupported on the analytics-engine route", + isAnalyticsParquetIndicesEnabled()); enableCalcite(); createTestIndexWithAliasFields(); loadIndex(Index.DATA_TYPE_ALIAS); From d076417b32baa3d5b5f2351e28372667b6c4a92d Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Thu, 11 Jun 2026 10:49:38 -0700 Subject: [PATCH 33/36] [analytics-engine] Keep binary in scan set; dedup strip type-list; preserve untouched bulk docs binary maps to VARBINARY in the analytics-engine schema builder (same as ip) and is a registered parquet field type, so it scans fine - drop it from the unsupported-field strip set, which is now {nested, geo_point, geo_shape, alias}. Stripping binary needlessly mutated fixtures and risked masking a real binary-handling bug instead of letting the assertion surface it. Make UNSUPPORTED_FIELD_TYPES the single source of truth (public) and have AnalyticsUnsupportedFieldStripVerifyIT import it instead of re-listing, so the stripper and the verifier cannot drift. Add a byte-identity guard to stripBulkFields: a bulk doc line is only re-serialized when it actually carried a dropped key, so untouched docs pass through verbatim. Signed-off-by: Eric Wei --- ...nalyticsUnsupportedFieldStripVerifyIT.java | 9 ++-- .../sql/legacy/AnalyticsFieldStripTests.java | 39 ++++++++++++--- .../org/opensearch/sql/legacy/TestUtils.java | 49 ++++++++++++++++--- 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java index 3a7a6feaa66..a1316ec1fda 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java @@ -25,8 +25,8 @@ * *

      *
    1. Create succeeded — the index exists (a mapping that still contained - * nested/geo_point/geo_shape/binary/alias would have been rejected by the parquet/composite - * store at PUT time, so existence proves the strip removed them). + * nested/geo_point/geo_shape/alias would have been rejected by the parquet/composite store at + * PUT time, so existence proves the strip removed them). *
    2. Mapping is clean — the live mapping pulled back from the cluster contains none of * the unsupported types at any depth. *
    3. Data agrees with mapping — every doc is searchable and no doc carries a stripped key @@ -42,8 +42,11 @@ */ public class AnalyticsUnsupportedFieldStripVerifyIT extends PPLIntegTestCase { + // Single source of truth — the same set the load-path strip uses + // (TestUtils.AnalyticsIndexConfig). + // Importing it (rather than re-listing) guarantees the verifier and the stripper can't drift. private static final Set UNSUPPORTED = - Set.of("nested", "geo_point", "geo_shape", "binary", "alias"); + org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig.UNSUPPORTED_FIELD_TYPES; /** * Field types the parquet/composite store also rejects but that are out of scope for the strip diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java index 557a79294ad..e14b74e3623 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -17,8 +17,9 @@ /** * Pure-logic coverage for the analytics-engine field strip (no filesystem / cluster). Verifies that - * nested/geo_point/geo_shape/binary/alias fields are removed from mappings and bulk data on the AE - * route, and that everything is a no-op when the route is off. + * nested/geo_point/geo_shape/alias fields are removed from mappings and bulk data on the AE route, + * that a supported {@code binary} field is left intact, and that everything is a no-op when the + * route is off. */ public class AnalyticsFieldStripTests { @@ -37,6 +38,7 @@ private void enable() { + "\"nested_value\":{\"type\":\"nested\"}," + "\"geo_point_value\":{\"type\":\"geo_point\"}," + "\"geo_shape_value\":{\"type\":\"geo_shape\"}," + // binary is a SUPPORTED scan type (binary -> VARBINARY); it must be kept, not stripped. + "\"binary_value\":{\"type\":\"binary\"}," + "\"alias_value\":{\"type\":\"alias\",\"path\":\"keep_text\"}," + "\"obj\":{\"properties\":{" @@ -51,15 +53,14 @@ public void mappingStrip_removesUnsupportedRecursively_andReportsTopLevel() { Set dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); assertEquals( - Set.of("nested_value", "geo_point_value", "geo_shape_value", "binary_value", "alias_value"), - dropped); + Set.of("nested_value", "geo_point_value", "geo_shape_value", "alias_value"), dropped); JSONObject props = json.getJSONObject("mappings").getJSONObject("properties"); assertTrue("supported scalar kept", props.has("keep_text")); + assertTrue("binary is a supported scan type — must be kept", props.has("binary_value")); assertFalse(props.has("nested_value")); assertFalse(props.has("geo_point_value")); assertFalse(props.has("geo_shape_value")); - assertFalse(props.has("binary_value")); assertFalse(props.has("alias_value")); // object field kept, but its unsupported sub-property stripped recursively @@ -82,12 +83,13 @@ public void bulkStrip_removesDroppedKeysFromSourceLinesOnly() { enable(); String bulk = "{\"index\":{\"_id\":\"1\"}}\n" - + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"binary_value\":\"AA==\"}\n" + + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"geo_shape_value\":\"POINT(1" + + " 2)\"}\n" + "{\"index\":{\"_id\":\"2\"}}\n" + "{\"keep_text\":\"y\",\"nested_value\":[{\"a\":1}]}\n"; String out = AnalyticsIndexConfig.stripBulkFields( - bulk, Set.of("geo_point_value", "binary_value", "nested_value")); + bulk, Set.of("geo_point_value", "geo_shape_value", "nested_value")); String[] lines = out.split("\n"); // action lines untouched @@ -97,12 +99,33 @@ public void bulkStrip_removesDroppedKeysFromSourceLinesOnly() { JSONObject doc1 = new JSONObject(lines[1]); assertTrue(doc1.has("keep_text")); assertFalse(doc1.has("geo_point_value")); - assertFalse(doc1.has("binary_value")); + assertFalse(doc1.has("geo_shape_value")); JSONObject doc2 = new JSONObject(lines[3]); assertTrue(doc2.has("keep_text")); assertFalse(doc2.has("nested_value")); } + @Test + public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { + enable(); + // doc1 carries a dropped key (gets rewritten); doc2 does not (must pass through verbatim). + String docWithDrop = "{\"keep_text\":\"x\",\"nested_value\":[{\"a\":1}]}"; + String docNoDrop = "{\"keep_text\":\"y\",\"age\": 30,\"z\":1}"; + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + docWithDrop + + "\n" + + "{\"index\":{\"_id\":\"2\"}}\n" + + docNoDrop + + "\n"; + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of("nested_value")); + String[] lines = out.split("\n", -1); + // The doc that had no dropped key is byte-for-byte identical (odd spacing/key order preserved). + assertEquals(docNoDrop, lines[3]); + // The doc that had a dropped key lost it. + assertFalse(new JSONObject(lines[1]).has("nested_value")); + } + @Test public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 2510e34bbd1..cb798dc31ff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -146,14 +146,34 @@ static String bulkLoadRefreshParam() { } /** - * Field types the analytics-engine (DataFusion) backend cannot read. Any property of one of - * these types is removed from the mapping (and the matching key from each bulk doc) before a - * test index is created/loaded on the analytics-engine route, so seeding a dataset that happens - * to use one of them doesn't fail the whole test. Applied uniformly to every dataset — no + * Field types the analytics-engine (DataFusion) backend cannot read, and which therefore must + * be removed from a test index's mapping (and the matching key from each bulk doc) before the + * index is created/loaded on the analytics-engine route. Seeding a dataset that happens to use + * one of them would otherwise fail ingestion and surface as an unrelated {@code expected:<1> + * but was:<0>} downstream — so we strip uniformly at load time across every dataset, no * per-index variant files to author or keep in sync. + * + *

      This set is the single source of truth. The strip triggers purely on a mapping's + * field types (not on which IT is running), so any IT — existing or newly added — that + * loads one of these datasets through {@link SQLIntegTestCase#loadIndex} is handled out of the + * box. {@code AnalyticsUnsupportedFieldStripVerifyIT} imports this same constant and proves no + * listed type survives in any live mapping, so the list cannot silently drift. + * + *

      What is and isn't here, and why: + * + *

        + *
      • {@code nested} — entire subtree dropped (the engine has no nested-document support). + *
      • {@code geo_point}, {@code geo_shape} — fall through the engine's type whitelist ({@code + * OpenSearchSchemaBuilder.mapFieldType} default → null), so the column can't scan. + *
      • {@code alias} — indirection the scan-row-type builder doesn't resolve. + *
      • {@code binary} is intentionally NOT here — the engine maps {@code binary → + * VARBINARY} (same as {@code ip}, see {@code OpenSearchSchemaBuilder.mapFieldType}), so + * it is a supported scan type. Stripping it would needlessly mutate fixtures and could + * mask a real binary-handling bug rather than let the assertion surface it. + *
      */ - private static final Set UNSUPPORTED_FIELD_TYPES = - Set.of("nested", "geo_point", "geo_shape", "binary", "alias"); + public static final Set UNSUPPORTED_FIELD_TYPES = + Set.of("nested", "geo_point", "geo_shape", "alias"); /** * Remove every {@link #UNSUPPORTED_FIELD_TYPES} property (recursively, including object @@ -209,6 +229,11 @@ private static boolean removeIfUnsupported(JSONObject properties, String field) * alternates an action line ({@code {"index":{...}}}) with a source line; only source lines * (those without a bulk action key) are rewritten. No-op when disabled or {@code droppedFields} * is empty. + * + *

      A source line is re-serialized only when it actually carried one of the dropped + * keys; every other line (action lines and docs that never had the field) is appended + * byte-for-byte unchanged. This keeps the payload as close to the on-disk fixture as possible — + * we don't reorder keys or reformat numbers on docs we aren't modifying. */ static String stripBulkFields(String bulkBody, Set droppedFields) { if (!isEnabled() || droppedFields.isEmpty()) { @@ -224,10 +249,18 @@ static String stripBulkFields(String bulkBody, Set droppedFields) { boolean isActionLine = doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); if (!isActionLine) { + boolean removedAny = false; for (String f : droppedFields) { - doc.remove(f); + if (doc.has(f)) { + doc.remove(f); + removedAny = true; + } + } + // Only rewrite the line if we actually removed something; otherwise leave it verbatim + // so untouched docs stay byte-for-byte identical to the fixture. + if (removedAny) { + line = doc.toString(); } - line = doc.toString(); } } out.append(line); From efdcc7ebd4b942458119d8b23bf4c3e1fc1879a1 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Thu, 11 Jun 2026 11:47:49 -0700 Subject: [PATCH 34/36] [analytics-engine] Exclude PPL ITs doomed by the unsupported-field strip The unsupported-field strip removes nested/geo_point/geo_shape/alias columns from test data+mappings on the AE route, so PPL tests whose queries reference such a column can never pass - the data is gone. Physically exclude them in integ-test/build.gradle (gated on tests.analytics.parquet_indices), centralized with the existing AE excludes and grouped comments, rather than letting them fail. Verified field-by-field against the index mappings; only doomed work is removed: - GeoPointFormatsIT (whole class - every test reads a geo_point field). - DataTypeIT.test_nonnumeric_data_types (nested_value + geo_point_value). - DataTypeIT.test_alias_data_type (where alias_col, type=alias). - SystemFunctionIT.typeof_opensearch_types (typeof(geo_point_value)). Cleared as NOT doomed (kept running): GeoIpFunctionsIT (geoip() over text ip/name), StatsCommandIT.testStatsNested* (nested function calls, not a nested field), ConvertCommandIT/TrendlineCommandIT/ExplainIT alias usages (result aliases, not alias-typed fields). Signed-off-by: Eric Wei --- integ-test/build.gradle | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index c8515f015a2..9f6cb3b050e 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1033,6 +1033,16 @@ task integTestRemote(type: RestIntegTestTask) { // to the analytics-engine planner via index settings. if (System.getProperty("tests.analytics.parquet_indices") != null) { systemProperty 'tests.analytics.parquet_indices', System.getProperty("tests.analytics.parquet_indices") + + // === Class-level excludes: PPL ITs doomed by the unsupported-field strip === + // On the analytics-engine route, TestUtils.AnalyticsIndexConfig strips + // nested/geo_point/geo_shape/alias fields from every test mapping AND its bulk data, because + // the parquet/composite store can't scan them. Tests whose queries reference such a field + // therefore can't pass — the data no longer exists. Exclude them here rather than asserting. + // + // GeoPointFormatsIT: EVERY test reads a geo_point field (point / location.point / nested geo + // in geopoint_index_mapping + complex_geo_index_mapping) — whole class is doomed. + exclude 'org/opensearch/sql/ppl/GeoPointFormatsIT.class' } // Primary-shard count for analytics-backed test indices (default 1). Set to e.g. 3 to @@ -1064,6 +1074,18 @@ task integTestRemote(type: RestIntegTestTask) { excludeTestsMatching '*OperatorIT.testGreaterOperator' excludeTestsMatching '*OperatorIT.testGteOperator' excludeTestsMatching '*OperatorIT.testNotOperator' + + // === Method-level excludes: PPL tests querying a stripped field === + // These reference a nested/geo_point/alias-typed field that the unsupported-field strip + // removes from the test data+mapping on the AE route, so they can't pass. (Verified + // field-by-field against the index mappings; only the doomed methods are excluded — the + // rest of each class still runs.) + // + // datatypes_index_mapping: nested_value=nested, geo_point_value=geo_point. + excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_nonnumeric_data_types' + excludeTestsMatching 'org.opensearch.sql.ppl.SystemFunctionIT.typeof_opensearch_types' + // alias_index_mapping: alias_col is type=alias; query is `where alias_col > 1`. + excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_alias_data_type' } } From ab4e7c6c71781644a56e7cd89cb6fe15f280659b Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Thu, 11 Jun 2026 15:26:36 -0700 Subject: [PATCH 35/36] [analytics-engine] Address review: path-aware strip, bulk-error fail-loud, stronger verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the correctness holes raised in review. Blocking 1 — path-aware mapping+source strip. stripUnsupportedMappingFields now returns exact dropped PATHS (e.g. [location, point]) instead of only top-level names; stripBulkFields removes those paths recursively from _source, descending through objects AND arrays of objects, preserving siblings (location.city etc.). Adds unit coverage for a complex_geo shape and arrays of objects. Blocking 2 — fail loud on bulk item failures. loadDataByRestClient now parses the _bulk response and throws on errors=true (naming the index + first item errors), for ALL test bulk loads. Silent partial ingestion no longer slips through a 200 status. Blocking 3 — verifier proves the invariant. AnalyticsUnsupportedFieldStripVerifyIT deletes each index first so the real load path always runs (not short-circuited by isIndexExist), then per dataset checks: clean live mapping, no stripped path left in sampled _source, and doc-count == source-doc count (tolerating a cluster-side count error on system indices with a transparent logged skip). NB1 — GeoPointFormatsIT exclude moved under the parsed analyticsEnabled gate (was gated on mere property presence, so =false wrongly excluded it). NB2 — out-of-scope skip (join) now refuses to skip if the mapping also declares any unsupported type, so a mixed failure can't be hidden. binary is now conditional: kept when store:true (engine reads VARBINARY), but stripped when store:false because the parquet store can't create it ("Unable to derive source for [X] with store disabled", verified on the AE cluster). Confirmed end-to-end: the verifier passes over all Index datasets on a force-routed AE cluster. Signed-off-by: Eric Wei --- integ-test/build.gradle | 23 +- ...nalyticsUnsupportedFieldStripVerifyIT.java | 308 ++++++++++++++++-- .../sql/legacy/AnalyticsFieldStripTests.java | 112 ++++++- .../org/opensearch/sql/legacy/TestUtils.java | 204 ++++++++---- 4 files changed, 530 insertions(+), 117 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 9f6cb3b050e..1f8ce9596f9 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1033,16 +1033,6 @@ task integTestRemote(type: RestIntegTestTask) { // to the analytics-engine planner via index settings. if (System.getProperty("tests.analytics.parquet_indices") != null) { systemProperty 'tests.analytics.parquet_indices', System.getProperty("tests.analytics.parquet_indices") - - // === Class-level excludes: PPL ITs doomed by the unsupported-field strip === - // On the analytics-engine route, TestUtils.AnalyticsIndexConfig strips - // nested/geo_point/geo_shape/alias fields from every test mapping AND its bulk data, because - // the parquet/composite store can't scan them. Tests whose queries reference such a field - // therefore can't pass — the data no longer exists. Exclude them here rather than asserting. - // - // GeoPointFormatsIT: EVERY test reads a geo_point field (point / location.point / nested geo - // in geopoint_index_mapping + complex_geo_index_mapping) — whole class is doomed. - exclude 'org/opensearch/sql/ppl/GeoPointFormatsIT.class' } // Primary-shard count for analytics-backed test indices (default 1). Set to e.g. 3 to @@ -1075,12 +1065,15 @@ task integTestRemote(type: RestIntegTestTask) { excludeTestsMatching '*OperatorIT.testGteOperator' excludeTestsMatching '*OperatorIT.testNotOperator' - // === Method-level excludes: PPL tests querying a stripped field === - // These reference a nested/geo_point/alias-typed field that the unsupported-field strip - // removes from the test data+mapping on the AE route, so they can't pass. (Verified - // field-by-field against the index mappings; only the doomed methods are excluded — the - // rest of each class still runs.) + // === Excludes: PPL ITs doomed by the unsupported-field strip === + // On the AE route TestUtils.AnalyticsIndexConfig strips nested/geo_point/geo_shape/alias + // fields from every test mapping AND its bulk data (the parquet/composite store can't + // scan them). A test whose query references such a field can't pass — the data is gone. + // Verified field-by-field against the index mappings; only doomed work is excluded. + // Gated on the parsed boolean so `-Dtests.analytics.parquet_indices=false` excludes none. // + // GeoPointFormatsIT: EVERY test reads a geo_point field — whole class doomed. + excludeTestsMatching 'org.opensearch.sql.ppl.GeoPointFormatsIT' // datatypes_index_mapping: nested_value=nested, geo_point_value=geo_point. excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_nonnumeric_data_types' excludeTestsMatching 'org.opensearch.sql.ppl.SystemFunctionIT.typeof_opensearch_types' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java index a1316ec1fda..a194b2999e2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java @@ -5,34 +5,45 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestUtils.analyticsDroppedFields; +import static org.opensearch.sql.legacy.TestUtils.getResourceFilePath; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Set; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.Assume; import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; /** * Exhaustive, fail-loud proof that data ingestion on the analytics-engine route is free of - * unsupported-field problems. For EVERY {@link Index} value, it runs the real {@link #loadIndex} - * harness path against the AE cluster, then verifies three independent legs: + * unsupported-field problems. For EVERY {@link Index} value it forces a clean (re)load through the + * real {@link #loadIndex} harness path against the AE cluster, then verifies, per dataset: * *

        - *
      1. Create succeeded — the index exists (a mapping that still contained - * nested/geo_point/geo_shape/alias would have been rejected by the parquet/composite store at - * PUT time, so existence proves the strip removed them). + *
      2. The real load path ran from a clean index — the index is deleted first, so {@code + * loadIndex}'s {@code if (!isIndexExist)} guard cannot short-circuit and inspect a stale + * index left by an earlier test. {@code loadDataByRestClient} now fails loudly on bulk {@code + * errors=true}, so a partial ingestion surfaces here as a load failure, not silently. *
      3. Mapping is clean — the live mapping pulled back from the cluster contains none of * the unsupported types at any depth. - *
      4. Data agrees with mapping — every doc is searchable and no doc carries a stripped key - * (a leftover key would dynamic-map the field back, re-introducing the unsupported type — we - * assert the live mapping stays clean after refresh and that doc count > 0 where data - * exists). + *
      5. Source agrees with mapping — no sampled {@code _source} doc still carries a stripped + * path (a leftover value would dynamic-map the field back, re-introducing the unsupported + * type). + *
      6. Doc count is sane — a fixture with N source docs ingests exactly N (proving no item + * silently dropped). Skipped only for a degenerate index whose strip left no scannable + * columns (e.g. cascaded_nested, all-{@code nested}), which the engine can't query at all. *
      * *

      This converts the "silent failure" risk (a missed strip surfacing only as an unrelated {@code @@ -42,16 +53,20 @@ */ public class AnalyticsUnsupportedFieldStripVerifyIT extends PPLIntegTestCase { + private static final Logger LOG = + LogManager.getLogger(AnalyticsUnsupportedFieldStripVerifyIT.class); + // Single source of truth — the same set the load-path strip uses - // (TestUtils.AnalyticsIndexConfig). - // Importing it (rather than re-listing) guarantees the verifier and the stripper can't drift. + // (TestUtils.AnalyticsIndexConfig). Importing it (not re-listing) keeps verifier and stripper in + // lockstep. private static final Set UNSUPPORTED = org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig.UNSUPPORTED_FIELD_TYPES; /** * Field types the parquet/composite store also rejects but that are out of scope for the strip - * (by product decision). An index whose creation fails solely because of one of these is skipped, - * not reported — this proof is strictly about {@link #UNSUPPORTED}. + * (by product decision). An index whose creation fails solely because of one of these is skipped + * — but only after confirming its mapping carries no {@link #UNSUPPORTED} type (see {@link + * #safeToSkipForOutOfScopeType}), so a mixed failure can't be hidden. */ private static final Set OUT_OF_SCOPE_TYPES = Set.of("join"); @@ -70,18 +85,28 @@ public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { List failures = new ArrayList<>(); for (Index index : Index.values()) { String name = index.getName(); + String mapping = index.getMapping(); + + // Force the real load path to run: delete any index a prior test left behind so loadIndex's + // `if (!isIndexExist)` guard can't short-circuit and inspect a stale index. + try { + deleteIndexIfExists(name); + } catch (IOException e) { + failures.add("[" + index.name() + " -> " + name + "] could not delete: " + rootMessage(e)); + continue; + } + try { loadIndex(index); } catch (Exception e) { - // A genuinely-absent dataset file is a pre-existing repo issue (dangling enum entry), not - // an - // unsupported-field ingestion problem — skip it so this stays a clean, attributable proof. if (isMissingDatasetFile(e)) { + // Dangling enum entry with no dataset file on disk — pre-existing repo issue, not an + // unsupported-field problem. Skip. continue; } - // An index that fails creation solely because of an out-of-scope type (e.g. join) is not - // part of the fix we're verifying — skip rather than report. - if (failsOnlyOnOutOfScopeType(e)) { + if (safeToSkipForOutOfScopeType(e, mapping)) { + // Creation failed solely on an out-of-scope type (e.g. join) AND the mapping carries no + // unsupported type we're responsible for — not our concern. Skip. continue; } failures.add( @@ -93,7 +118,8 @@ public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { + rootMessage(e)); continue; } - // Leg 1+2: index exists and its live mapping carries no unsupported type at any depth. + + // Leg 2: live mapping carries no unsupported type at any depth. List offending = unsupportedFieldsInLiveMapping(name); if (!offending.isEmpty()) { failures.add( @@ -104,6 +130,55 @@ public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { + "] live mapping still has unsupported fields: " + offending); } + + // Leg 3: no sampled _source doc still carries a stripped path. + Set> dropped = analyticsDroppedFields(mapping); + List leftover = strippedPathsStillInSource(name, dropped); + if (!leftover.isEmpty()) { + failures.add( + "[" + + index.name() + + " -> " + + name + + "] _source still carries stripped paths " + + leftover + + " (dynamic mapping would re-introduce the unsupported type)"); + } + + // Leg 4: a non-empty fixture ingests exactly its source-doc count — proving no item silently + // dropped. Skipped for a degenerate index whose strip left no scannable columns (can't + // query). + // The count goes through PPL `stats count()`; if that query errors for a cluster-side reason + // unrelated to the strip (e.g. it transiently touches a parquet-backed system index), we + // record a transparent skip rather than fail — Legs 1-3 plus the load-time bulk errors=false + // check (TestUtils.loadDataByRestClient) already catch partial ingestion directly. + int expectedDocs = sourceDocCount(index); + if (expectedDocs > 0 && liveMappingHasColumns(name)) { + Integer actualDocs = liveDocCountOrNull(name); + if (actualDocs == null) { + skippedDocCount.add(index.name() + " (count query errored — see Legs 1-3 + bulk check)"); + } else if (actualDocs != expectedDocs) { + failures.add( + "[" + + index.name() + + " -> " + + name + + "] ingested " + + actualDocs + + " docs but dataset" + + " has " + + expectedDocs + + " (partial/failed ingestion)"); + } + } + } + + // Surface skipped doc-count checks so coverage gaps are visible, not silent. + if (!skippedDocCount.isEmpty()) { + LOG.info( + "Leg 4 (doc-count) skipped for {} dataset(s): {}", + skippedDocCount.size(), + String.join(", ", skippedDocCount)); } if (!failures.isEmpty()) { @@ -115,20 +190,155 @@ public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { } } + private final List skippedDocCount = new ArrayList<>(); + + private void deleteIndexIfExists(String indexName) throws IOException { + try { + client().performRequest(new Request("DELETE", "/" + indexName)); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 404) { + throw e; + } + } + } + /** Pull the live mapping back from the cluster and collect any unsupported-typed field paths. */ private List unsupportedFieldsInLiveMapping(String indexName) throws IOException { - Request request = new Request("GET", "/" + indexName + "/_mapping"); - Response response = client().performRequest(request); - JSONObject body = new JSONObject(getResponseBody(response)); - // shape: { indexName: { mappings: { properties: {...} } } } - JSONObject mappings = body.getJSONObject(indexName).getJSONObject("mappings"); List offending = new ArrayList<>(); - if (mappings.has("properties")) { - collectUnsupported(mappings.getJSONObject("properties"), "", offending); + JSONObject props = liveMappingProperties(indexName); + if (props != null) { + collectUnsupported(props, "", offending); } return offending; } + /** + * True when the index's live mapping still has at least one top-level property. An index whose + * mapping was reduced to nothing by the strip (all-unsupported dataset, e.g. cascaded_nested) has + * no scannable columns and can't be queried on the AE route — Leg 4 is skipped for it. + */ + private boolean liveMappingHasColumns(String indexName) throws IOException { + JSONObject props = liveMappingProperties(indexName); + return props != null && !props.isEmpty(); + } + + /** {@code mappings.properties} of the live index, or null when the mapping has no properties. */ + private JSONObject liveMappingProperties(String indexName) throws IOException { + Response response = client().performRequest(new Request("GET", "/" + indexName + "/_mapping")); + JSONObject mappings = + new JSONObject(getResponseBody(response)) + .getJSONObject(indexName) + .getJSONObject("mappings"); + return mappings.has("properties") ? mappings.getJSONObject("properties") : null; + } + + /** + * Sample up to a few docs via the search API and report any {@code dropped} path that still + * exists in a returned {@code _source}. A leftover value is what would let dynamic mapping + * re-create the unsupported field after refresh. + */ + private List strippedPathsStillInSource(String indexName, Set> dropped) + throws IOException { + List leftover = new ArrayList<>(); + if (dropped.isEmpty()) { + return leftover; + } + Request search = new Request("POST", "/" + indexName + "/_search?size=50"); + Response response; + try { + response = client().performRequest(search); + } catch (ResponseException e) { + // A degenerate index (no columns) can't be searched; nothing to check. + return leftover; + } + JSONObject body = new JSONObject(getResponseBody(response)); + JSONArray hits = body.getJSONObject("hits").getJSONArray("hits"); + for (int i = 0; i < hits.length(); i++) { + JSONObject source = hits.getJSONObject(i).optJSONObject("_source"); + if (source == null) { + continue; + } + for (List path : dropped) { + if (pathPresent(source, path, 0)) { + String dotted = String.join(".", path); + if (!leftover.contains(dotted)) { + leftover.add(dotted); + } + } + } + } + return leftover; + } + + /** Mirror of TestUtils.removePath traversal: is {@code path[idx..]} present in {@code node}? */ + private boolean pathPresent(Object node, List path, int idx) { + String part = path.get(idx); + boolean last = idx == path.size() - 1; + if (node instanceof JSONObject) { + JSONObject obj = (JSONObject) node; + if (!obj.has(part)) { + return false; + } + return last || pathPresent(obj.get(part), path, idx + 1); + } else if (node instanceof JSONArray) { + JSONArray arr = (JSONArray) node; + for (int j = 0; j < arr.length(); j++) { + if (pathPresent(arr.get(j), path, idx)) { + return true; + } + } + } + return false; + } + + /** + * Live doc count of the index, obtained through the analytics-engine query path ({@code + * source=idx | stats count()}). The REST {@code _count} API is NOT usable here — on a + * parquet-backed {@code DataFormatAwareEngine} index it fails with "Cannot apply function ... + * directly on IndexShard". Counting via PPL also exercises the real scan path, so the result + * proves the data both ingested and is readable on the AE route. + */ + private Integer liveDocCountOrNull(String indexName) { + String query = String.format("source=%s | stats count() as c", indexName); + try { + JSONObject result = executeQuery(query); + return result.getJSONArray("datarows").getJSONArray(0).getInt(0); + } catch (Exception e) { + // The count query can fail for cluster-side reasons unrelated to the strip (e.g. it touches a + // parquet-backed system index that can't serve a shard-level count). Don't let that mask the + // strip proof — return null so the caller records a transparent skip. + LOG.info("count query failed for [{}]: {}", indexName, rootMessage(e)); + return null; + } + } + + /** + * Number of source (non-action) lines in a dataset's bulk NDJSON file — i.e. how many docs the + * load is expected to ingest. Returns 0 when the index has no dataset file (mapping-only enum + * entries), which makes Leg 4 a no-op for those. + */ + private static int sourceDocCount(Index index) throws IOException { + String dataSet = index.getDataSet(); + if (dataSet == null || dataSet.isEmpty()) { + return 0; + } + String body = new String(Files.readAllBytes(Paths.get(getResourceFilePath(dataSet)))); + int docs = 0; + for (String line : body.split("\n", -1)) { + String trimmed = line.trim(); + if (trimmed.isEmpty() || trimmed.charAt(0) != '{') { + continue; + } + JSONObject obj = new JSONObject(trimmed); + boolean isActionLine = + obj.has("index") || obj.has("create") || obj.has("update") || obj.has("delete"); + if (!isActionLine) { + docs++; + } + } + return docs; + } + private void collectUnsupported(JSONObject properties, String prefix, List out) { for (String field : properties.keySet()) { JSONObject def = properties.optJSONObject(field); @@ -147,17 +357,53 @@ private void collectUnsupported(JSONObject properties, String prefix, Listnested paths and arrays of objects — that a supported {@code binary} field is + * left intact, and that everything is a no-op when the route is off. */ public class AnalyticsFieldStripTests { @@ -32,14 +33,20 @@ private void enable() { System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); } + private static List path(String... parts) { + return List.of(parts); + } + private static final String MAPPING = "{\"mappings\":{\"properties\":{" + "\"keep_text\":{\"type\":\"text\"}," + "\"nested_value\":{\"type\":\"nested\"}," + "\"geo_point_value\":{\"type\":\"geo_point\"}," + "\"geo_shape_value\":{\"type\":\"geo_shape\"}," - // binary is a SUPPORTED scan type (binary -> VARBINARY); it must be kept, not stripped. + // default binary (store:false) — parquet store can't create it → stripped. + "\"binary_value\":{\"type\":\"binary\"}," + // binary with store:true — engine reads it as VARBINARY → kept. + + "\"binary_stored\":{\"type\":\"binary\",\"store\":true}," + "\"alias_value\":{\"type\":\"alias\",\"path\":\"keep_text\"}," + "\"obj\":{\"properties\":{" + " \"keep_inner\":{\"type\":\"keyword\"}," @@ -47,17 +54,28 @@ private void enable() { + "}}}"; @Test - public void mappingStrip_removesUnsupportedRecursively_andReportsTopLevel() { + public void mappingStrip_removesUnsupportedRecursively_andReportsExactPaths() { enable(); JSONObject json = new JSONObject(MAPPING); - Set dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + Set> dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); assertEquals( - Set.of("nested_value", "geo_point_value", "geo_shape_value", "alias_value"), dropped); + Set.of( + path("nested_value"), + path("geo_point_value"), + path("geo_shape_value"), + // default binary (store:false) is dropped — parquet store can't create it. + path("binary_value"), + path("alias_value"), + // recursive: the geo_point under the object reports its full path. + path("obj", "inner_geo")), + dropped); JSONObject props = json.getJSONObject("mappings").getJSONObject("properties"); assertTrue("supported scalar kept", props.has("keep_text")); - assertTrue("binary is a supported scan type — must be kept", props.has("binary_value")); + assertTrue("store:true binary is scannable — must be kept", props.has("binary_stored")); + assertFalse( + "store:false binary not creatable on parquet store — dropped", props.has("binary_value")); assertFalse(props.has("nested_value")); assertFalse(props.has("geo_point_value")); assertFalse(props.has("geo_shape_value")); @@ -70,16 +88,84 @@ public void mappingStrip_removesUnsupportedRecursively_andReportsTopLevel() { assertFalse("nested geo_point inside object dropped", inner.has("inner_geo")); } + /** + * complex_geo-shaped coverage (the case the top-level-only strip used to miss): a {@code + * geo_point} buried under object fields. Mapping must drop the exact nested path; bulk source + * must drop the same nested value while keeping unaffected siblings. + */ + @Test + public void complexGeoShape_stripsNestedPath_andKeepsSiblings() { + enable(); + String mapping = + "{\"mappings\":{\"properties\":{" + + "\"location\":{\"properties\":{" + + " \"point\":{\"type\":\"geo_point\"}," + + " \"name\":{\"type\":\"text\"}," + + " \"city\":{\"type\":\"keyword\"}," + + " \"country\":{\"type\":\"keyword\"}}}," + + "\"nested_locations\":{\"properties\":{" + + " \"primary\":{\"properties\":{\"office\":{\"type\":\"geo_point\"}}}}}" + + "}}}"; + JSONObject json = new JSONObject(mapping); + Set> dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + + // exact paths, not top-level names + assertEquals( + Set.of(path("location", "point"), path("nested_locations", "primary", "office")), dropped); + + JSONObject loc = json.getJSONObject("mappings").getJSONObject("properties"); + JSONObject locProps = loc.getJSONObject("location").getJSONObject("properties"); + assertFalse("location.point removed", locProps.has("point")); + assertTrue("location.name kept", locProps.has("name")); + assertTrue("location.city kept", locProps.has("city")); + assertTrue("location.country kept", locProps.has("country")); + + // now strip a matching source doc by those exact paths + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"location\":{\"point\":{\"lat\":1,\"lon\":2},\"name\":\"hq\",\"city\":\"sea\"," + + "\"country\":\"us\"}," + + "\"nested_locations\":{\"primary\":{\"office\":{\"lat\":3,\"lon\":4}}}}\n"; + String out = AnalyticsIndexConfig.stripBulkFields(bulk, dropped); + JSONObject doc = new JSONObject(out.split("\n")[1]); + JSONObject docLoc = doc.getJSONObject("location"); + assertFalse("source location.point removed", docLoc.has("point")); + assertTrue("source location.name kept", docLoc.has("name")); + assertTrue("source location.city kept", docLoc.has("city")); + assertTrue("source location.country kept", docLoc.has("country")); + assertFalse( + "source nested_locations.primary.office removed", + doc.getJSONObject("nested_locations").getJSONObject("primary").has("office")); + } + + /** A dropped path may sit under an array of objects — every element must be stripped. */ + @Test + public void bulkStrip_handlesArraysOfObjects() { + enable(); + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"offices\":[{\"loc\":{\"lat\":1},\"name\":\"a\"},{\"loc\":{\"lat\":2}," + + "\"name\":\"b\"}]}\n"; + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("offices", "loc"))); + JSONObject doc = new JSONObject(out.split("\n")[1]); + var arr = doc.getJSONArray("offices"); + for (int i = 0; i < arr.length(); i++) { + JSONObject office = arr.getJSONObject(i); + assertFalse("offices[" + i + "].loc removed", office.has("loc")); + assertTrue("offices[" + i + "].name kept", office.has("name")); + } + } + @Test public void mappingStrip_noopWhenDisabled() { JSONObject json = new JSONObject(MAPPING); - Set dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + Set> dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); assertTrue(dropped.isEmpty()); assertTrue(json.getJSONObject("mappings").getJSONObject("properties").has("nested_value")); } @Test - public void bulkStrip_removesDroppedKeysFromSourceLinesOnly() { + public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { enable(); String bulk = "{\"index\":{\"_id\":\"1\"}}\n" @@ -89,7 +175,7 @@ public void bulkStrip_removesDroppedKeysFromSourceLinesOnly() { + "{\"keep_text\":\"y\",\"nested_value\":[{\"a\":1}]}\n"; String out = AnalyticsIndexConfig.stripBulkFields( - bulk, Set.of("geo_point_value", "geo_shape_value", "nested_value")); + bulk, Set.of(path("geo_point_value"), path("geo_shape_value"), path("nested_value"))); String[] lines = out.split("\n"); // action lines untouched @@ -118,7 +204,7 @@ public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { + "{\"index\":{\"_id\":\"2\"}}\n" + docNoDrop + "\n"; - String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of("nested_value")); + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("nested_value"))); String[] lines = out.split("\n", -1); // The doc that had no dropped key is byte-for-byte identical (odd spacing/key order preserved). assertEquals(docNoDrop, lines[3]); @@ -130,7 +216,7 @@ public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; // disabled -> unchanged even with a drop set - assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of("geo_point_value"))); + assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("geo_point_value")))); // enabled but empty drop set -> unchanged enable(); assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of())); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index cb798dc31ff..c478165bf07 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -159,6 +159,9 @@ static String bulkLoadRefreshParam() { * box. {@code AnalyticsUnsupportedFieldStripVerifyIT} imports this same constant and proves no * listed type survives in any live mapping, so the list cannot silently drift. * + *

      This set is the always-strip core. {@code binary} is handled conditionally and is + * NOT in this set — see {@link #isUnsupportedForAeStore}. + * *

      What is and isn't here, and why: * *

        @@ -166,24 +169,50 @@ static String bulkLoadRefreshParam() { *
      • {@code geo_point}, {@code geo_shape} — fall through the engine's type whitelist ({@code * OpenSearchSchemaBuilder.mapFieldType} default → null), so the column can't scan. *
      • {@code alias} — indirection the scan-row-type builder doesn't resolve. - *
      • {@code binary} is intentionally NOT here — the engine maps {@code binary → - * VARBINARY} (same as {@code ip}, see {@code OpenSearchSchemaBuilder.mapFieldType}), so - * it is a supported scan type. Stripping it would needlessly mutate fixtures and could - * mask a real binary-handling bug rather than let the assertion surface it. + *
      • {@code binary} is conditional, not in this set. The engine's read path maps + * {@code binary → VARBINARY} (same as {@code ip}), so a binary column scans fine — but + * the parquet/composite store rejects creating a {@code binary} field that lacks + * {@code store: true} ("Unable to derive source for [X] with store disabled", verified on + * the AE cluster). So we strip a {@code binary} field only when it is NOT {@code store: + * true}; a {@code store: true} binary field is kept and ingests/scans normally. Adding + * {@code binary} to this unconditional set would over-strip the supported case. *
      */ public static final Set UNSUPPORTED_FIELD_TYPES = Set.of("nested", "geo_point", "geo_shape", "alias"); + /** + * Whether a field definition cannot be created/scanned on the analytics-engine route and so + * must be stripped. True for any {@link #UNSUPPORTED_FIELD_TYPES} type, and additionally for a + * {@code binary} field that is not {@code store: true} — the parquet store can't derive {@code + * _source} for a store-disabled binary field at mapping time. + */ + private static boolean isUnsupportedForAeStore(JSONObject def) { + String type = def.optString("type", null); + if (type == null) { + return false; + } + if (UNSUPPORTED_FIELD_TYPES.contains(type)) { + return true; + } + return "binary".equals(type) && !def.optBoolean("store", false); + } + /** * Remove every {@link #UNSUPPORTED_FIELD_TYPES} property (recursively, including object - * sub-properties) from an index-creation JSON's {@code mappings.properties}, and report the set - * of top-level field names that were dropped so the matching keys can be stripped from + * sub-properties) from an index-creation JSON's {@code mappings.properties}, and report the + * exact dotted paths that were dropped so the matching values can be stripped from * bulk data. No-op when disabled. Mutates {@code jsonObject} in place. * - * @return the top-level property names removed (empty when disabled or nothing matched) + *

      Paths are returned as ordered part lists, e.g. {@code ["location", "point"]} for a {@code + * geo_point} sub-field of an object {@code location}, so the bulk-source strip can remove the + * exact nested value while preserving unaffected siblings ({@code location.city} etc.). A + * top-level unsupported field comes back as a single-element list, e.g. {@code + * ["nested_value"]}. + * + * @return the dropped field paths (empty when disabled or nothing matched) */ - static Set stripUnsupportedMappingFields(JSONObject jsonObject) { + static Set> stripUnsupportedMappingFields(JSONObject jsonObject) { if (!isEnabled() || !jsonObject.has("mappings")) { return Set.of(); } @@ -191,52 +220,48 @@ static Set stripUnsupportedMappingFields(JSONObject jsonObject) { if (!mappings.has("properties")) { return Set.of(); } - Set droppedTopLevel = new HashSet<>(); - JSONObject properties = mappings.getJSONObject("properties"); - for (String field : properties.keySet().toArray(new String[0])) { - if (removeIfUnsupported(properties, field)) { - droppedTopLevel.add(field); - } - } - return droppedTopLevel; + Set> dropped = new HashSet<>(); + collectAndRemoveUnsupported(mappings.getJSONObject("properties"), new ArrayList<>(), dropped); + return dropped; } /** - * If {@code properties[field]} is an unsupported type, remove it and return true. Otherwise - * recurse into its nested {@code properties} (a plain {@code object} field) and keep it. + * Walk a {@code properties} block, removing every property whose {@code type} is unsupported + * (recording its full path) and recursing into object sub-{@code properties} for the rest. */ - private static boolean removeIfUnsupported(JSONObject properties, String field) { - JSONObject def = properties.optJSONObject(field); - if (def == null) { - return false; - } - String type = def.optString("type", null); - if (type != null && UNSUPPORTED_FIELD_TYPES.contains(type)) { - properties.remove(field); - return true; - } - if (def.has("properties")) { - JSONObject sub = def.getJSONObject("properties"); - for (String child : sub.keySet().toArray(new String[0])) { - removeIfUnsupported(sub, child); + private static void collectAndRemoveUnsupported( + JSONObject properties, List prefix, Set> dropped) { + for (String field : properties.keySet().toArray(new String[0])) { + JSONObject def = properties.optJSONObject(field); + if (def == null) { + continue; + } + List path = new ArrayList<>(prefix); + path.add(field); + if (isUnsupportedForAeStore(def)) { + properties.remove(field); + dropped.add(List.copyOf(path)); + } else if (def.has("properties")) { + collectAndRemoveUnsupported(def.getJSONObject("properties"), path, dropped); } } - return false; } /** - * Strip the given top-level keys from every document line of a bulk NDJSON payload. Bulk format - * alternates an action line ({@code {"index":{...}}}) with a source line; only source lines - * (those without a bulk action key) are rewritten. No-op when disabled or {@code droppedFields} - * is empty. + * Strip the given dropped paths from every source document of a bulk NDJSON payload. + * Bulk format alternates an action line ({@code {"index":{...}}}) with a source line; only + * source lines (those without a bulk action key) are rewritten. No-op when disabled or {@code + * droppedPaths} is empty. * - *

      A source line is re-serialized only when it actually carried one of the dropped - * keys; every other line (action lines and docs that never had the field) is appended - * byte-for-byte unchanged. This keeps the payload as close to the on-disk fixture as possible — - * we don't reorder keys or reformat numbers on docs we aren't modifying. + *

      Each path is removed recursively: it descends through nested objects and arrays of + * objects (so a {@code nested}/object array has the field stripped from every element), + * leaving unaffected siblings intact. A source line is re-serialized only when removing a + * path actually changed it; every other line (action lines and docs that never had the + * dropped path) is appended byte-for-byte unchanged, so untouched docs match the fixture + * exactly. */ - static String stripBulkFields(String bulkBody, Set droppedFields) { - if (!isEnabled() || droppedFields.isEmpty()) { + static String stripBulkFields(String bulkBody, Set> droppedPaths) { + if (!isEnabled() || droppedPaths.isEmpty()) { return bulkBody; } String[] lines = bulkBody.split("\n", -1); @@ -250,11 +275,8 @@ static String stripBulkFields(String bulkBody, Set droppedFields) { doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); if (!isActionLine) { boolean removedAny = false; - for (String f : droppedFields) { - if (doc.has(f)) { - doc.remove(f); - removedAny = true; - } + for (List path : droppedPaths) { + removedAny |= removePath(doc, path, 0); } // Only rewrite the line if we actually removed something; otherwise leave it verbatim // so untouched docs stay byte-for-byte identical to the fixture. @@ -271,6 +293,37 @@ static String stripBulkFields(String bulkBody, Set droppedFields) { return out.toString(); } + /** + * Remove {@code path[idx..]} from {@code node}, descending through objects and arrays of + * objects. Returns true if anything was removed. At the last path part the key is deleted from + * the object; otherwise we recurse into the child object (or each object element of a child + * array). + */ + private static boolean removePath(Object node, List path, int idx) { + String part = path.get(idx); + boolean last = idx == path.size() - 1; + boolean removed = false; + if (node instanceof JSONObject) { + JSONObject obj = (JSONObject) node; + if (!obj.has(part)) { + return false; + } + if (last) { + obj.remove(part); + return true; + } + removed = removePath(obj.get(part), path, idx + 1); + } else if (node instanceof JSONArray) { + // The mapping path can sit under an array (e.g. a nested/object array); strip from each + // object element, ignoring non-object elements. + JSONArray arr = (JSONArray) node; + for (int j = 0; j < arr.length(); j++) { + removed |= removePath(arr.get(j), path, idx); + } + } + return removed; + } + private AnalyticsIndexConfig() {} } @@ -294,12 +347,13 @@ public static void createIndexByRestClient(RestClient client, String indexName, } /** - * Top-level field names the analytics-engine route would strip from {@code mapping}. Used by - * {@link #loadDataByRestClient(RestClient, String, String, java.util.Set)} so bulk docs drop the - * same keys the mapping dropped. Returns an empty set when AE is disabled or {@code mapping} is + * Exact dropped field paths the analytics-engine route would remove from {@code mapping} + * (e.g. {@code ["location", "point"]} for a nested {@code geo_point}). Used by {@link + * #loadDataByRestClient(RestClient, String, String, java.util.Set)} so bulk docs drop the same + * paths the mapping dropped. Returns an empty set when AE is disabled or {@code mapping} is * empty. */ - public static Set analyticsDroppedFields(String mapping) { + public static Set> analyticsDroppedFields(String mapping) { if (isNullOrEmpty(mapping)) { return Set.of(); } @@ -374,21 +428,55 @@ public static void loadDataByRestClient( /** * Same as {@link #loadDataByRestClient(RestClient, String, String)} but strips {@code - * droppedFields} (the keys removed from the mapping on the analytics-engine route) from every - * bulk source doc, so the index mapping and the data agree. When AE is disabled or {@code - * droppedFields} is empty this is byte-for-byte identical to the 3-arg form. + * droppedPaths} (the exact field paths removed from the mapping on the analytics-engine route) + * from every bulk source doc, so the index mapping and the data agree. When AE is disabled or + * {@code droppedPaths} is empty this is byte-for-byte identical to the 3-arg form. */ public static void loadDataByRestClient( - RestClient client, String indexName, String dataSetFilePath, Set droppedFields) + RestClient client, String indexName, String dataSetFilePath, Set> droppedPaths) throws IOException { Path path = Paths.get(getResourceFilePath(dataSetFilePath)); String body = new String(Files.readAllBytes(path)); - body = AnalyticsIndexConfig.stripBulkFields(body, droppedFields); + body = AnalyticsIndexConfig.stripBulkFields(body, droppedPaths); Request request = new Request( "POST", "/" + indexName + "/_bulk?" + AnalyticsIndexConfig.bulkLoadRefreshParam()); request.setJsonEntity(body); - performRequest(client, request); + Response response = performRequest(client, request); + failIfBulkHadItemErrors(indexName, response); + } + + /** + * A {@code _bulk} call can return HTTP 200 while individual items failed ({@code "errors":true}). + * That silent partial ingestion is exactly the failure mode that surfaces later as an unrelated + * row-count/assertion error in some downstream IT, so fail loudly here — naming the index and the + * first few item errors — for ALL test bulk loads, not just the analytics-engine route. + */ + private static void failIfBulkHadItemErrors(String indexName, Response response) + throws IOException { + JSONObject body = new JSONObject(getResponseBody(response)); + if (!body.optBoolean("errors", false)) { + return; + } + JSONArray items = body.optJSONArray("items"); + List firstErrors = new ArrayList<>(); + if (items != null) { + for (int i = 0; i < items.length() && firstErrors.size() < 5; i++) { + JSONObject action = items.getJSONObject(i); + // each item is { "": { ..., "error": {...} } } where is index/create/update/delete + for (String op : action.keySet()) { + JSONObject result = action.optJSONObject(op); + if (result != null && result.has("error")) { + firstErrors.add("doc#" + i + ": " + result.get("error")); + } + } + } + } + throw new IllegalStateException( + "Bulk load into [" + + indexName + + "] had item failures (errors=true). First failures:\n " + + String.join("\n ", firstErrors)); } /** From f649962bc6de39649f319f7162176db3433e40e4 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Thu, 11 Jun 2026 17:14:02 -0700 Subject: [PATCH 36/36] [analytics-engine] Centralize CalciteAliasFieldAggregationIT exclude in build.gradle Per review: keep all AE-route test exclusions in one place. Move the inline Assume.assumeFalse(isAnalyticsParquetIndicesEnabled()) guard out of CalciteAliasFieldAggregationIT into the gated analyticsEnabled exclude block in integ-test/build.gradle, alongside the other doomed PPL ITs. The rationale (raw-PUT alias index can't be created on the AE route) is carried into both the build.gradle comment and a short note left at the test's init(). Signed-off-by: Eric Wei --- integ-test/build.gradle | 3 +++ .../calcite/remote/CalciteAliasFieldAggregationIT.java | 10 +++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1f8ce9596f9..db32fcb2c2b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -1079,6 +1079,9 @@ task integTestRemote(type: RestIntegTestTask) { excludeTestsMatching 'org.opensearch.sql.ppl.SystemFunctionIT.typeof_opensearch_types' // alias_index_mapping: alias_col is type=alias; query is `where alias_col > 1`. excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_alias_data_type' + // CalciteAliasFieldAggregationIT: raw-PUT alias index can't be created on the AE route + // and every test queries alias fields directly — whole class doomed. + excludeTestsMatching 'org.opensearch.sql.calcite.remote.CalciteAliasFieldAggregationIT' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java index 6fb5cfcd0ee..5a23492ca6e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -15,7 +15,6 @@ import java.io.IOException; import java.util.List; import org.json.JSONObject; -import org.junit.Assume; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.client.ResponseException; @@ -32,12 +31,9 @@ public class CalciteAliasFieldAggregationIT extends PPLIntegTestCase { @Override public void init() throws Exception { super.init(); - // Alias fields are unsupported on the analytics-engine (parquet/composite) route — the index - // can't even be created, and these tests query the alias fields directly. There's nothing to - // salvage by stripping, so skip the whole class on the AE route. Runs normally otherwise. - Assume.assumeFalse( - "Alias-field aggregation is unsupported on the analytics-engine route", - isAnalyticsParquetIndicesEnabled()); + // Excluded on the analytics-engine route from integ-test/build.gradle (alias fields are + // unsupported there — the raw-PUT index can't even be created and these tests query the alias + // fields directly), alongside the other AE-route exclusions kept in one place. enableCalcite(); createTestIndexWithAliasFields(); loadIndex(Index.DATA_TYPE_ALIAS);