diff --git a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java index c1a9e74b763..a84300e65f8 100644 --- a/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java +++ b/api/src/main/java/org/opensearch/sql/api/UnifiedQueryPlanner.java @@ -14,6 +14,7 @@ import org.apache.calcite.rel.RelRoot; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.logical.LogicalSort; +import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.util.SqlVisitor; @@ -25,6 +26,7 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.exception.QueryEngineException; import org.opensearch.sql.exception.SemanticCheckException; @@ -74,13 +76,18 @@ public RelNode plan(String query) { } catch (SyntaxCheckException | QueryEngineException | UnsupportedOperationException - | IllegalArgumentException e) { - LOG.error("Failed to plan query: {}", e.getMessage()); + | IllegalArgumentException + | ErrorReport e) { + LOG.warn("Failed to plan query: {}", e.getMessage()); throw e; + } catch (CalciteException e) { + // Calcite validation errors (e.g. table not found) indicate an invalid query. + LOG.warn("Failed to plan query, invalid query: {}", e.getMessage()); + throw new SemanticCheckException(e.getMessage(), e); } catch (AssertionError e) { // Calcite throws assertion error directly when building bad RelNode String message = "Failed to plan query: invalid plan structure"; - LOG.error(message, e); + LOG.warn(message, e); throw new SemanticCheckException(message, e); } catch (Exception e) { String message = "Failed to plan query: unexpected error"; diff --git a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java index d6280b829c7..d60ef7ed4ec 100644 --- a/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java +++ b/api/src/main/java/org/opensearch/sql/api/parser/SqlV2QueryParser.java @@ -8,8 +8,11 @@ import static org.opensearch.sql.ast.dsl.AstDSL.existsSubquery; import static org.opensearch.sql.ast.dsl.AstDSL.inSubquery; import static org.opensearch.sql.ast.dsl.AstDSL.join; +import static org.opensearch.sql.ast.dsl.AstDSL.union; +import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import org.antlr.v4.runtime.tree.ParseTree; import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -22,6 +25,7 @@ import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ExistsSubqueryExpressionAtomContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.InSubqueryPredicateContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.JoinClauseContext; +import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.UnionSelectContext; import org.opensearch.sql.sql.parser.AstBuilder; import org.opensearch.sql.sql.parser.AstExpressionBuilder; import org.opensearch.sql.sql.parser.AstStatementBuilder; @@ -81,6 +85,13 @@ private JoinType toJoinType(JoinClauseContext ctx) { }; } + @Override + public UnresolvedPlan visitUnionSelect(UnionSelectContext ctx) { + List datasets = + ctx.querySpecification().stream().map(this::visit).collect(Collectors.toList()); + return union(datasets); + } + /** * Expression builder with IN/EXISTS subquery support. Accesses the enclosing AstBuilder to * visit subquery plan nodes. Must be created via {@link #createExpressionBuilder()} because the diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java index b818795609f..db3d517f0b4 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerSqlV2Test.java @@ -200,6 +200,42 @@ WHERE age NOT IN (SELECT age FROM catalog.departments WHERE dept_name = 'Enginee """); } + @Test + public void testUnionAll() { + givenQuery( + """ + SELECT name FROM catalog.employees UNION ALL SELECT dept_name FROM catalog.departments + """) + .assertPlan( + """ + LogicalUnion(all=[true]) + LogicalProject(name=[$1]) + LogicalTableScan(table=[[catalog, employees]]) + LogicalProject(dept_name=[$1]) + LogicalTableScan(table=[[catalog, departments]]) + """); + } + + @Test + public void testMultiWayUnion() { + givenQuery( + """ + SELECT name FROM catalog.employees + UNION ALL SELECT dept_name FROM catalog.departments + UNION ALL SELECT name FROM catalog.employees + """) + .assertPlan( + """ + LogicalUnion(all=[true]) + LogicalProject(name=[$1]) + LogicalTableScan(table=[[catalog, employees]]) + LogicalProject(dept_name=[$1]) + LogicalTableScan(table=[[catalog, departments]]) + LogicalProject(name=[$1]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + @Test public void testNotExistsSubquery() { givenQuery( @@ -241,6 +277,57 @@ public void selectExpressionWithoutFrom() { """); } + @Test + public void testGroupByAggregateAlias() { + givenQuery( + """ + SELECT department, SUM(age) AS total FROM catalog.employees GROUP BY department + """) + .assertPlan( + """ + LogicalProject(department=[$0], total=[$1]) + LogicalAggregate(group=[{0}], SUM(age)=[SUM($1)]) + LogicalProject(department=[$3], age=[$2]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testOrderByAggregateAlias() { + givenQuery( + """ + SELECT department, COUNT(*) AS cnt FROM catalog.employees + GROUP BY department ORDER BY cnt DESC LIMIT 3 + """) + .assertPlan( + """ + LogicalSort(sort0=[$1], dir0=[DESC-nulls-last]) + LogicalProject(department=[$1], cnt=[$0]) + LogicalSort(sort0=[$0], dir0=[DESC-nulls-last], fetch=[3]) + LogicalProject(COUNT(*)=[$1], department=[$0]) + LogicalAggregate(group=[{0}], COUNT(*)=[COUNT()]) + LogicalProject(department=[$3]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testAliasPreservedInOutputSchema() { + givenQuery("SELECT COUNT(*) AS cnt FROM catalog.employees").assertFields("cnt"); + + givenQuery("SELECT department, COUNT(*) AS cnt FROM catalog.employees GROUP BY department") + .assertFields("department", "cnt"); + + givenQuery("SELECT department, COUNT(*) FROM catalog.employees GROUP BY department") + .assertFields("department", "COUNT(*)"); + + givenQuery("SELECT MAX(age) + MIN(age) AS range_sum FROM catalog.employees") + .assertFields("range_sum"); + + givenQuery("SELECT id, name, age AS years, department FROM catalog.employees") + .assertFields("id", "name", "years", "department"); + } + @Test public void testHavingMaxCol() { givenQuery( @@ -259,6 +346,47 @@ GROUP BY department HAVING MAX(age) > 30 """); } + @Test + public void testCountStarWithFilter() { + givenQuery("SELECT COUNT(*) FILTER(WHERE age > 30) FROM catalog.employees") + .assertPlan( + """ + LogicalAggregate(group=[{}], COUNT(*) FILTER(WHERE age > 30)=[COUNT() FILTER $0]) + LogicalProject($f1=[>($2, 30)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testFilteredAggregateWithGroupBy() { + givenQuery( + """ + SELECT department, SUM(age) FILTER(WHERE age > 30) FROM catalog.employees + GROUP BY department + """) + .assertPlan( + """ + LogicalAggregate(group=[{0}], SUM(age) FILTER(WHERE age > 30)=[SUM($1) FILTER $2]) + LogicalProject(department=[$3], age=[$2], $f3=[>($2, 30)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testMultipleFilteredAggregates() { + givenQuery( + """ + SELECT MAX(age) FILTER(WHERE age > 30), MIN(age) FILTER(WHERE age < 50) + FROM catalog.employees + """) + .assertPlan( + """ + LogicalAggregate(group=[{}], MAX(age) FILTER(WHERE age > 30)=[MAX($0) FILTER $1], MIN(age) FILTER(WHERE age < 50)=[MIN($0) FILTER $2]) + LogicalProject(age=[$2], $f4=[>($2, 30)], $f5=[<($2, 50)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + @Test public void testScalarFnOverAggregate() { givenQuery("SELECT ABS(MAX(age)) FROM catalog.employees") @@ -337,6 +465,33 @@ GROUP BY department HAVING MAX(age) > 30 AND MIN(age) < 50 """); } + @Test + public void testCountDistinctWindowWithOrderBy() { + // No frame printed: RANGE .. CURRENT ROW is Calcite's default for ORDER BY. + givenQuery( + """ + SELECT department, COUNT(DISTINCT name) OVER(ORDER BY department) FROM catalog.employees + """) + .assertPlan( + """ + LogicalProject(department=[$3], COUNT(DISTINCT name) OVER(ORDER BY department)=[COUNT(DISTINCT $1) OVER (ORDER BY $3 NULLS FIRST)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + + @Test + public void testSumWindowWithPartitionAndOrderBy() { + givenQuery( + """ + SELECT name, SUM(age) OVER(PARTITION BY department ORDER BY age) FROM catalog.employees + """) + .assertPlan( + """ + LogicalProject(name=[$1], SUM(age) OVER(PARTITION BY department ORDER BY age)=[SUM($2) OVER (PARTITION BY $3 ORDER BY $2 NULLS FIRST)]) + LogicalTableScan(table=[[catalog, employees]]) + """); + } + @Test public void testWindowOrderByDefaultsNullsFirst() { // Window function ORDER BY without explicit NULLS FIRST/LAST defaults to NULLS FIRST, diff --git a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java index bf9ff38d694..296e9eb2519 100644 --- a/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java +++ b/api/src/test/java/org/opensearch/sql/api/UnifiedQueryPlannerTest.java @@ -10,10 +10,12 @@ import java.util.Map; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.runtime.CalciteException; import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.impl.AbstractSchema; import org.junit.Test; import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.executor.QueryType; @@ -72,7 +74,7 @@ public void testPPLQueryPlanningWithDefaultNamespaceMultiLevel() { // This is valid in SparkSQL, but Calcite requires "catalog" as the default root schema to // resolve it - assertThrows(IllegalStateException.class, () -> planner.plan("source = opensearch.employees")); + assertThrows(SemanticCheckException.class, () -> planner.plan("source = opensearch.employees")); } @Test @@ -131,6 +133,20 @@ public void semanticErrorIsRethrownAsSemanticCheckException() { .assertErrorMessageEquals("Source and target patterns have different wildcard counts"); } + @Test + public void fieldNotFoundIsRethrownAsErrorReport() { + givenInvalidQuery("source = catalog.employees | where unknown_field = 1") + .assertErrorType(ErrorReport.class) + .assertErrorMessageContains("Field [unknown_field] not found"); + } + + @Test + public void invalidTableIsRethrownAsSemanticCheckException() { + givenInvalidQuery("source = catalog.nonexistent_table") + .assertErrorType(SemanticCheckException.class) + .assertCauseType(CalciteException.class); + } + @Test public void assertionErrorIsWrappedAsSemanticCheckException() { // Remove when the underlying Calcite assertion is fixed. diff --git a/build.gradle b/build.gradle index d892c31e1c4..7b76532aad0 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.7.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.8.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/build.gradle b/core/build.gradle index 4c7cfb34235..4d0d98edb29 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -65,11 +65,11 @@ dependencies { } api 'org.apache.calcite:calcite-linq4j:1.41.0' api project(':common') - compileOnly 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT' + compileOnly 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' // Needed because analytics-api's QueryPlanExecutor signature uses // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type. compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" - testImplementation 'org.opensearch.sandbox:analytics-api:3.7.0-SNAPSHOT' + testImplementation 'org.opensearch.sandbox:analytics-api:3.8.0-SNAPSHOT' testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}" implementation "com.github.seancfoley:ipaddress:5.4.2" implementation "com.jayway.jsonpath:json-path:2.9.0" diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index f00a87ab40f..3a96137f42d 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -84,6 +84,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.Union; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -781,4 +782,8 @@ public static InSubquery inSubquery(UnresolvedPlan query, UnresolvedExpression.. public static ExistsSubquery existsSubquery(UnresolvedPlan query) { return new ExistsSubquery(query); } + + public static Union union(List datasets) { + return new Union(datasets); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java b/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java index d4241165cbc..62c6559a225 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/WindowBound.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ast.expression; +import lombok.EqualsAndHashCode; import lombok.Getter; public abstract class WindowBound { @@ -25,6 +26,7 @@ public boolean isPreceding() { } } + @EqualsAndHashCode(callSuper = false) public static class CurrentRowWindowBound extends WindowBound { CurrentRowWindowBound() {} diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java b/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java index 7ea1a072f8a..9e2fbf246d2 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/WindowFrame.java @@ -40,6 +40,13 @@ public static WindowFrame toCurrentRow() { AstDSL.stringLiteral("CURRENT ROW")); } + public static WindowFrame rangeToCurrentRow() { + return WindowFrame.of( + FrameType.RANGE, + AstDSL.stringLiteral("UNBOUNDED PRECEDING"), + AstDSL.stringLiteral("CURRENT ROW")); + } + public static WindowFrame of(FrameType type, String lower, String upper) { return WindowFrame.of(type, AstDSL.stringLiteral(lower), AstDSL.stringLiteral(upper)); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Union.java b/core/src/main/java/org/opensearch/sql/ast/tree/Union.java index a96831567cb..7eefe89ce53 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/Union.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Union.java @@ -21,15 +21,25 @@ @RequiredArgsConstructor @AllArgsConstructor public class Union extends UnresolvedPlan { + /** Input subplans (operands) combined by this UNION ALL. */ private final List datasets; + /** Whether inputs are unified to a common schema by name (PPL) vs combined positionally (SQL). */ + private boolean unifySchema; + + /** Optional cap on output rows (PPL {@code maxout}); {@code null} if unbounded. */ private Integer maxout; + /** PPL constructor: UNION ALL with schema unification. */ + public Union(List datasets, Integer maxout) { + this(datasets, true, maxout); + } + @Override public UnresolvedPlan attach(UnresolvedPlan child) { List newDatasets = ImmutableList.builder().add(child).addAll(datasets).build(); - return new Union(newDatasets, maxout); + return new Union(newDatasets, unifySchema, maxout); } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java index 0512316628c..e7a5a5a68bd 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteAggCallVisitor.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.tools.RelBuilder.AggCall; import org.apache.logging.log4j.util.Strings; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -46,10 +47,17 @@ public AggCall visitAggregateFunction(AggregateFunction node, CalcitePlanContext } return BuiltinFunctionName.ofAggregation(node.getFuncName()) .map( - functionName -> { - return PlanUtils.makeAggCall( - context, functionName, node.getDistinct(), field, argList); - }) + functionName -> + PlanUtils.makeAggCall(context, functionName, node.getDistinct(), field, argList)) + // Apply the optional FILTER(WHERE ...) predicate; IS TRUE treats NULL as non-matching. + .map( + aggCall -> + node.condition() == null + ? aggCall + : aggCall.filter( + context.rexBuilder.makeCall( + SqlStdOperatorTable.IS_TRUE, + rexNodeVisitor.analyze(node.condition(), context)))) .orElseThrow( () -> new UnsupportedOperationException("Unexpected aggregation: " + node.getFuncName())); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 335b48a45d9..ff2e9fbd4bd 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -481,11 +481,30 @@ public RelNode visitProject(Project node, CalcitePlanContext context) { if (!context.isResolvingSubquery()) { context.setProjectVisited(true); } - context.relBuilder.project(expandedFields); + + // Force the projection on a rename: without it, Calcite omits the project node when the + // columns are unchanged (same fields and order), so an alias like COUNT(*) AS cnt is lost. + boolean force = isRenameFieldsProject(expandedFields, currentFields); + context.relBuilder.project(expandedFields, ImmutableList.of(), force); } return context.relBuilder.peek(); } + private static boolean isRenameFieldsProject(List fields, List currentFields) { + for (RexNode r : fields) { + if (r.getKind() == AS) { + RexCall as = (RexCall) r; + if (as.getOperands().get(0) instanceof RexInputRef ref) { + String name = ((RexLiteral) as.getOperands().get(1)).getValueAs(String.class); + if (!name.equals(currentFields.get(ref.getIndex()))) { + return true; + } + } + } + } + return false; + } + private boolean isSingleAllFieldsProject(Project node) { return node.getProjectList().size() == 1 && node.getProjectList().getFirst() instanceof AllFields; @@ -915,6 +934,9 @@ public RelNode visitTranspose( RelBuilder b = context.relBuilder; RexBuilder rx = context.rexBuilder; RelDataType varchar = rx.getTypeFactory().createSqlType(SqlTypeName.VARCHAR); + int axisLiteralLength = fieldNames.stream().mapToInt(String::length).max().orElse(0); + RelDataType axisLiteralType = + rx.getTypeFactory().createSqlType(SqlTypeName.CHAR, axisLiteralLength); // Step 1: ROW_NUMBER b.projectPlus( @@ -932,18 +954,22 @@ public RelNode visitTranspose( .map( f -> Map.entry( - ImmutableList.of(rx.makeLiteral(f)), + ImmutableList.of( + (RexLiteral) rx.makeLiteral(f, axisLiteralType, false, false)), ImmutableList.of((RexNode) rx.makeCast(varchar, b.field(f), true)))) .collect(Collectors.toList())); // Step 3: Trim spaces from columnName column before pivot RexNode trimmedColumnName = - context.rexBuilder.makeCall( - SqlStdOperatorTable.TRIM, - context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), - context.rexBuilder.makeLiteral(" "), - b.field(columnName)); + context.rexBuilder.makeCast( + varchar, + context.rexBuilder.makeCall( + SqlStdOperatorTable.TRIM, + context.rexBuilder.makeFlag(SqlTrimFunction.Flag.BOTH), + context.rexBuilder.makeLiteral(" "), + b.field(columnName)), + true); // Step 4: PIVOT b.pivot( @@ -1537,6 +1563,7 @@ private Pair, List> aggregateWithTrimming( List aggCallRefs = PlanUtils.getInputRefsFromAggCall(resolvedAggCallList); boolean hintNestedAgg = containsNestedAggregator(context.relBuilder, aggCallRefs); trimmedRefs.addAll(aggCallRefs); + trimmedRefs.addAll(getAggCallFilterRefs(aggExprList, context)); context.relBuilder.project(trimmedRefs); // Re-resolve all attributes based on adding trimmed Project. @@ -1769,6 +1796,21 @@ private static AggregateFunction extractAggregateFunction(UnresolvedExpression e return null; } + /** + * Collects input refs used by aggregate FILTER(WHERE ...) predicates so trimming retains them. + */ + private List getAggCallFilterRefs( + List aggExprList, CalcitePlanContext context) { + List refs = new ArrayList<>(); + for (UnresolvedExpression aggExpr : aggExprList) { + AggregateFunction aggFunc = extractAggregateFunction(aggExpr); + if (aggFunc != null && aggFunc.condition() != null) { + refs.addAll(PlanUtils.getInputRefs(rexVisitor.analyze(aggFunc.condition(), context))); + } + } + return refs; + } + private Optional getTimeSpanField(UnresolvedExpression expr) { if (Objects.isNull(expr)) return Optional.empty(); if (expr instanceof Span span && SpanUnit.isTimeUnit(span.getUnit())) { @@ -3024,8 +3066,10 @@ public RelNode visitUnion(Union node, CalcitePlanContext context) { "Union command requires at least two datasets. Provided: " + inputNodes.size()); } - List unifiedInputs = - SchemaUnifier.buildUnifiedSchemaWithTypeCoercion(inputNodes, context); + List unifiedInputs = inputNodes; + if (node.isUnifySchema()) { + unifiedInputs = SchemaUnifier.buildUnifiedSchemaWithTypeCoercion(inputNodes, context); + } for (RelNode input : unifiedInputs) { context.relBuilder.push(input); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 830b40d2551..1d473b168e4 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; import javax.annotation.Nullable; @@ -84,11 +85,13 @@ import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.calcite.utils.SubsearchUtils; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.CoercionUtils; import org.opensearch.sql.expression.function.PPLFuncImpTable; @RequiredArgsConstructor @@ -225,24 +228,41 @@ public RexNode visitIn(In node, CalcitePlanContext context) { final RexNode field = analyze(node.getField(), context); final List valueList = node.getValueList().stream().map(value -> analyze(value, context)).toList(); + // When the field is a temporal type, do NOT use leastRestrictive + rexBuilder.makeIn. For a + // temporal field tested against string/date literals, leastRestrictive collapses the common + // type to VARCHAR (the EXPR_DATE / EXPR_TIMESTAMP UDTs are VARCHAR-backed), so makeIn casts the + // field DOWN to VARCHAR and string-compares mismatched renderings (e.g. '2018-06-23 00:00:00' + // against '2018-06-23') — silently matching nothing. Rewrite the membership test as an OR of + // PPL `=` comparisons, the same temporal-aware comparison path visitCompare takes for `=`, so + // each value is coerced to the field's timestamp domain before comparison. + ExprType fieldExprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(field.getType()); + if (TEMPORAL_TYPES.contains(fieldExprType)) { + List equalities = + valueList.stream() + .map(value -> PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, "=", field, value)) + .toList(); + return context.relBuilder.or(equalities); + } final List dataTypes = - new java.util.ArrayList<>(valueList.stream().map(RexNode::getType).toList()); + new ArrayList<>(valueList.stream().map(RexNode::getType).toList()); dataTypes.add(field.getType()); RelDataType commonType = context.rexBuilder.getTypeFactory().leastRestrictive(dataTypes); if (commonType != null) { List newValueList = valueList.stream().map(value -> context.rexBuilder.makeCast(commonType, value)).toList(); return context.rexBuilder.makeIn(field, newValueList); - } else { - List exprTypes = - dataTypes.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); - throw new SemanticCheckException( - StringUtils.format( - "In expression types are incompatible: fields type %s, values type %s", - exprTypes.getLast(), exprTypes.subList(0, exprTypes.size() - 1))); } + List exprTypes = + dataTypes.stream().map(OpenSearchTypeFactory::convertRelDataTypeToExprType).toList(); + throw new SemanticCheckException( + StringUtils.format( + "In expression types are incompatible: fields type %s, values type %s", + exprTypes.getLast(), exprTypes.subList(0, exprTypes.size() - 1))); } + private static final Set TEMPORAL_TYPES = + Set.of(ExprCoreType.DATE, ExprCoreType.TIME, ExprCoreType.TIMESTAMP); + @Override public RexNode visitCompare(Compare node, CalcitePlanContext context) { RexNode left = analyze(node.getLeft(), context); @@ -258,6 +278,25 @@ public RexNode visitCompare(Compare node, CalcitePlanContext context) { return PPLFuncImpTable.INSTANCE.resolve(context.rexBuilder, op, left, right); } + /** + * Widens a set of operands to a common temporal type when, and only when, every operand is a + * temporal type (DATE / TIME / TIMESTAMP), including the EXPR_DATE / EXPR_TIME / EXPR_TIMESTAMP + * UDTs. Returns {@code null} otherwise so non-temporal incompatible mixes still fail the type + * check. The widening reuses {@link CoercionUtils#widenArguments} — the same path comparison + * operators take — which resolves DATE / TIME to TIMESTAMP via the shared widening graph. + */ + private static @Nullable List widenTemporalOperands( + CalcitePlanContext context, List operands) { + boolean allTemporal = + operands.stream() + .map(node -> OpenSearchTypeFactory.convertRelDataTypeToExprType(node.getType())) + .allMatch(TEMPORAL_TYPES::contains); + if (!allTemporal) { + return null; + } + return CoercionUtils.widenArguments(context.rexBuilder, operands); + } + @Override public RexNode visitBetween(Between node, CalcitePlanContext context) { RexNode value = analyze(node.getValue(), context); @@ -268,12 +307,25 @@ public RexNode visitBetween(Between node, CalcitePlanContext context) { lowerBound = context.rexBuilder.makeCast(commonType, lowerBound); upperBound = context.rexBuilder.makeCast(commonType, upperBound); } else { - throw new SemanticCheckException( - StringUtils.format( - "BETWEEN expression types are incompatible: [%s, %s, %s]", - OpenSearchTypeFactory.convertRelDataTypeToExprType(value.getType()), - OpenSearchTypeFactory.convertRelDataTypeToExprType(lowerBound.getType()), - OpenSearchTypeFactory.convertRelDataTypeToExprType(upperBound.getType()))); + // leastRestrictive() has no common type for mixed temporal representations — e.g. a standard + // Calcite TIMESTAMP field compared against EXPR_DATE UDT bounds (`ts between date('...') and + // date('...')`). Comparison operators coerce these through CoercionUtils; BETWEEN calls + // leastRestrictive directly and would otherwise reject them. Fall back to the same temporal + // widening, scoped to all-temporal operands so genuinely incompatible mixes (e.g. + // `age between '35' and 38.5`) still raise SemanticCheckException. + List widened = + widenTemporalOperands(context, List.of(value, lowerBound, upperBound)); + if (widened == null) { + throw new SemanticCheckException( + StringUtils.format( + "BETWEEN expression types are incompatible: [%s, %s, %s]", + OpenSearchTypeFactory.convertRelDataTypeToExprType(value.getType()), + OpenSearchTypeFactory.convertRelDataTypeToExprType(lowerBound.getType()), + OpenSearchTypeFactory.convertRelDataTypeToExprType(upperBound.getType()))); + } + value = widened.get(0); + lowerBound = widened.get(1); + upperBound = widened.get(2); } return context.relBuilder.between(value, lowerBound, upperBound); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java index 11eabfd483c..676b1e9a776 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/rule/PPLSimplifyDedupRule.java @@ -6,11 +6,14 @@ package org.opensearch.sql.calcite.plan.rule; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.plan.RelRule; import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; @@ -22,6 +25,7 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; import org.apache.calcite.rex.RexWindow; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; @@ -115,8 +119,41 @@ protected void apply( RelCollation inputCollation = extractCollationFromWindow(windows.get(0)); + // Split the bucket-non-null filter into two parts: + // 1) IS_NOT_NULL conjuncts on a partition key — these are the bucket-non-null guards PPL + // emits as part of the dedup pattern; LogicalDedup absorbs their semantics. + // 2) Everything else — for example, a user `where` predicate that FilterMergeRule may + // have folded into the same conjunction, or a user IS_NOT_NULL filter on a non-partition + // column. These must be preserved as a separate filter below the new LogicalDedup so + // user-visible behavior is unchanged regardless of whether FilterMergeRule fired. + Set partitionKeyIndices = new HashSet<>(); + for (RexNode key : dedupColumns) { + if (key instanceof RexInputRef ref) { + partitionKeyIndices.add(ref.getIndex()); + } + } + List bucketNonNullConjuncts = new ArrayList<>(); + List remainingConjuncts = new ArrayList<>(); + for (RexNode conjunct : RelOptUtil.conjunctions(bucketNonNullFilter.getCondition())) { + if (isNotNullOnPartitionKey(conjunct, partitionKeyIndices)) { + bucketNonNullConjuncts.add(conjunct); + } else { + remainingConjuncts.add(conjunct); + } + } + // Defensive: if no IS_NOT_NULL conjunct on a partition key is present, this filter is not + // actually a bucket-non-null filter — bail out without transforming. The loose operand + // predicate may have matched on an unrelated AND that happens to contain an IS_NOT_NULL on + // some other ref. + if (bucketNonNullConjuncts.isEmpty()) { + return; + } + RelBuilder relBuilder = call.builder(); relBuilder.push(bucketNonNullFilter.getInput()); + if (!remainingConjuncts.isEmpty()) { + relBuilder.filter(RexUtil.composeConjunction(relBuilder.getRexBuilder(), remainingConjuncts)); + } List> targetProjections = projectWithWindow.getNamedProjects().stream() .filter(p -> !p.getKey().isA(SqlKind.ROW_NUMBER)) @@ -134,6 +171,14 @@ protected void apply( call.transformTo(relBuilder.build()); } + private static boolean isNotNullOnPartitionKey(RexNode rex, Set partitionKeyIndices) { + if (!PlanUtils.isNotNullOnRef(rex)) { + return false; + } + RexInputRef ref = (RexInputRef) ((RexCall) rex).getOperands().get(0); + return partitionKeyIndices.contains(ref.getIndex()); + } + private static @Nullable RelCollation extractCollationFromWindow(RexWindow window) { if (window.orderKeys.isEmpty()) { return null; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 9c15c1485c1..b8ed9b6cd7e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -47,7 +47,9 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.analytics.schema.BinaryType; +import org.opensearch.analytics.schema.DateOnlyType; import org.opensearch.analytics.schema.IpType; +import org.opensearch.analytics.schema.TimeOnlyType; import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.type.ExprBinaryType; import org.opensearch.sql.calcite.type.ExprDateType; @@ -277,6 +279,16 @@ public static ExprType convertRelDataTypeToExprType(RelDataType type) { return exprType; } + /** DATE check for return-type inference; recognizes the analytics-route {@link DateOnlyType}. */ + public static boolean isDateExprType(RelDataType type) { + return type instanceof DateOnlyType || convertRelDataTypeToExprType(type) == ExprCoreType.DATE; + } + + /** TIME counterpart of {@link #isDateExprType}; recognizes {@link TimeOnlyType}. */ + public static boolean isTimeExprType(RelDataType type) { + return type instanceof TimeOnlyType || convertRelDataTypeToExprType(type) == ExprCoreType.TIME; + } + /** * Result-schema-only variant of {@link #convertRelDataTypeToExprType} that recognizes the * analytics-engine {@link IpType} / {@link BinaryType} markers as {@link ExprCoreType#IP} / @@ -293,6 +305,13 @@ public static ExprType convertAnalyticsEngineRelDataTypeToExprType(RelDataType t if (type instanceof BinaryType) { return BINARY; } + // span() over date / time UDT — schema label is DATE / TIME, not TIMESTAMP. + if (type instanceof DateOnlyType) { + return DATE; + } + if (type instanceof TimeOnlyType) { + return TIME; + } return convertRelDataTypeToExprType(type); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java index acf2a6b8ebc..ce7dd408e7a 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLReturnTypes.java @@ -12,7 +12,6 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.type.SqlTypeUtil; -import org.opensearch.sql.data.type.ExprCoreType; /** * Return types used in PPL. This class complements the {@link @@ -36,8 +35,7 @@ private PPLReturnTypes() {} public static SqlReturnTypeInference TIME_APPLY_RETURN_TYPE = opBinding -> { RelDataType temporalType = opBinding.getOperandType(0); - if (ExprCoreType.TIME.equals( - OpenSearchTypeFactory.convertRelDataTypeToExprType(temporalType))) { + if (OpenSearchTypeFactory.isTimeExprType(temporalType)) { return UserDefinedFunctionUtils.NULLABLE_TIME_UDT; } return UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 84832071857..f899f747421 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -788,15 +788,21 @@ static Mapping mapping(List rexNodes, RelDataType schema) { return Mappings.target(getSelectColumns(rexNodes), schema.getFieldCount()); } + /** + * Accepts the un-merged {@code IS NOT NULL($ref)} shape and the merged-{@code AND} shape that + * {@link org.apache.calcite.rel.rules.FilterMergeRule} produces when a user {@code where} + * precedes the dedup. The concrete partition-key match — and the split-out of any user predicate + * folded into the AND — happens in {@link PPLSimplifyDedupRule#apply}. + */ static boolean mayBeFilterFromBucketNonNull(LogicalFilter filter) { RexNode condition = filter.getCondition(); return isNotNullOnRef(condition) || (condition instanceof RexCall rexCall && rexCall.getOperator().equals(SqlStdOperatorTable.AND) - && rexCall.getOperands().stream().allMatch(PlanUtils::isNotNullOnRef)); + && rexCall.getOperands().stream().anyMatch(PlanUtils::isNotNullOnRef)); } - private static boolean isNotNullOnRef(RexNode rex) { + public static boolean isNotNullOnRef(RexNode rex) { return rex instanceof RexCall rexCall && rexCall.isA(SqlKind.IS_NOT_NULL) && rexCall.getOperands().get(0) instanceof RexInputRef; diff --git a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java index da8eae41355..2a5d392a149 100644 --- a/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/ExecutionEngine.java @@ -13,6 +13,7 @@ import lombok.RequiredArgsConstructor; import lombok.ToString; import org.apache.calcite.rel.RelNode; +import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.common.response.ResponseListener; @@ -80,6 +81,8 @@ class QueryResponse { private final Schema schema; private final List results; private final Cursor cursor; + @lombok.Setter private QueryProfile profile; + @lombok.Setter private Throwable error; } @Data diff --git a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java index 51854f16fb7..68dffacc2eb 100644 --- a/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java +++ b/core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java @@ -12,13 +12,17 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.opensearch.analytics.exec.QueryPlanExecutor; +import org.opensearch.analytics.exec.profile.ProfiledResult; import org.opensearch.analytics.schema.BinaryType; +import org.opensearch.analytics.schema.DateOnlyType; import org.opensearch.analytics.schema.IpType; +import org.opensearch.analytics.schema.TimeOnlyType; import org.opensearch.common.network.InetAddresses; import org.opensearch.core.action.ActionListener; import org.opensearch.sql.ast.statement.ExplainMode; @@ -46,6 +50,15 @@ */ public class AnalyticsExecutionEngine implements ExecutionEngine { + // TIME-typed list elements round-trip via Timestamp and bypass ArrowValues' scalar + // post-processing (see DataFusion list_merge), arriving as "1970-01-01[ T]HH:mm:ss[.frac]". + private static final Pattern EPOCH_DATE_TIME_PREFIX = + Pattern.compile("^1970-01-01[ T](\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?)$"); + + // DateOnlyType wire is Timestamp(ms) at midnight — keep the date, drop the time. + private static final Pattern DATE_WITH_MIDNIGHT_TIME = + Pattern.compile("^(\\d{4}-\\d{2}-\\d{2})[ T]\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?$"); + private final QueryPlanExecutor> planExecutor; public AnalyticsExecutionEngine(QueryPlanExecutor> planExecutor) { @@ -145,6 +158,52 @@ public void explain( } } + /** + * Executes the query with profiling enabled. Returns results + stage timing profile. Called when + * {@code profile=true} is set on the request. + */ + public void executeWithProfile( + RelNode plan, + CalcitePlanContext context, + org.opensearch.analytics.QueryRequestContext queryCtx, + ResponseListener listener) { + + planExecutor.executeWithProfile( + plan, + queryCtx, + new ActionListener<>() { + @Override + public void onResponse(ProfiledResult result) { + try { + // ProfiledResult delivers the profile on BOTH success and failure paths + // so users get stage timing visibility even when a query partially fails. + QueryResponse response = buildProfiledResponse(plan, result); + listener.onResponse(response); + } catch (Exception e) { + listener.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private QueryResponse buildProfiledResponse(RelNode plan, ProfiledResult result) { + List fields = plan.getRowType().getFieldList(); + Schema schema = buildSchema(fields); + List results = + result.rows() != null ? convertRows(result.rows(), fields) : List.of(); + QueryResponse response = new QueryResponse(schema, results, Cursor.None); + response.setProfile(result.profile()); + if (!result.isSuccess()) { + response.setError(result.failure()); + } + return response; + } + private List convertRows(Iterable rows, List fields) { List results = new ArrayList<>(); for (Object[] row : rows) { @@ -159,21 +218,7 @@ private List convertRows(Iterable rows, List - *
  • {@link IpType} + {@code byte[]} → canonical address string (matches {@code - * IpFieldMapper}'s {@code valueFetcher} output). - *
  • {@link BinaryType} + {@code byte[]} → base64-encoded string (matches the OpenSearch - * {@code binary} field wire format). - *
  • Anything else → existing {@link ExprValueUtils#fromObjectValue} path. - * - * - *

    Without this dispatch, {@code fromObjectValue} throws {@code unsupported object class [B} on - * byte[] cells, and IP buffers leak through as raw 16-byte ipv4-mapped-ipv6 garbage. - */ + /** Renders UDT cells (IP/binary byte[]; date / time string) and strips TIME prefixes in lists. */ private static ExprValue toExprValue(Object value, RelDataType type) { if (value instanceof byte[] bytes) { if (type instanceof IpType) { @@ -187,9 +232,39 @@ private static ExprValue toExprValue(Object value, RelDataType type) { return ExprValueUtils.stringValue(Base64.getEncoder().encodeToString(bytes)); } } + // DateOnlyType scalar — strip midnight time suffix off the Timestamp(ms) wire. + if (type instanceof DateOnlyType && value instanceof String s) { + var m = DATE_WITH_MIDNIGHT_TIME.matcher(s); + if (m.matches()) { + return ExprValueUtils.stringValue(m.group(1)); + } + } + // TimeOnlyType scalar — strip 1970-01-01 prefix off the Timestamp(ms) wire. + if (type instanceof TimeOnlyType && value instanceof String s) { + var m = EPOCH_DATE_TIME_PREFIX.matcher(s); + if (m.matches()) { + return ExprValueUtils.stringValue(m.group(1)); + } + } + if (value instanceof List list) { + return ExprValueUtils.collectionValue(stripEpochDatePrefixInList(list)); + } return ExprValueUtils.fromObjectValue(value); } + private static List stripEpochDatePrefixInList(List list) { + List out = new ArrayList<>(list.size()); + for (Object element : list) { + if (element instanceof String s) { + var m = EPOCH_DATE_TIME_PREFIX.matcher(s); + out.add(m.matches() ? m.group(1) : s); + } else { + out.add(element); + } + } + return out; + } + private Schema buildSchema(List fields) { List columns = new ArrayList<>(); for (RelDataTypeField field : fields) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 60172e70c84..a24015de992 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -521,7 +521,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlAggFunction DISTINCT_COUNT_APPROX = createUserDefinedAggFunction( DistinctCountApproxLogicalAggFunction.class, - "DISTINCT_COUNT_APPROX", + // Substrait-standard name the analytics-engine backend resolves by (V3 overrides it). + "APPROX_COUNT_DISTINCT", ReturnTypes.BIGINT_FORCE_NULLABLE, null); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java index 9456e6b857a..2c040918e80 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/datetime/AddSubDateFunction.java @@ -63,7 +63,7 @@ public SqlReturnTypeInference getReturnTypeInference() { return opBinding -> { RelDataType temporalType = opBinding.getOperandType(0); RelDataType temporalDeltaType = opBinding.getOperandType(1); - if (OpenSearchTypeFactory.convertRelDataTypeToExprType(temporalType) == ExprCoreType.DATE + if (OpenSearchTypeFactory.isDateExprType(temporalType) && SqlTypeFamily.NUMERIC.contains(temporalDeltaType)) { return NULLABLE_DATE_UDT; } else { diff --git a/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java b/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java index e759b128a45..08b88048468 100644 --- a/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java @@ -30,7 +30,9 @@ import org.junit.jupiter.api.Test; import org.opensearch.analytics.exec.QueryPlanExecutor; import org.opensearch.analytics.schema.BinaryType; +import org.opensearch.analytics.schema.DateOnlyType; import org.opensearch.analytics.schema.IpType; +import org.opensearch.analytics.schema.TimeOnlyType; import org.opensearch.core.action.ActionListener; import org.opensearch.sql.calcite.CalcitePlanContext; import org.opensearch.sql.calcite.SysLimit; @@ -237,6 +239,67 @@ void executeRelNode_binaryColumnRendersAsBase64() { "byte[] should base64-encode to match OpenSearch binary wire format. " + dump); } + /** DateOnlyType — schema reports DATE, value strips midnight suffix. */ + @Test + void executeRelNode_dateOnlyTypeStripsTimeSuffix() { + RelNode relNode = + mockRelNodeWithType("d", new DateOnlyType(RelDataTypeSystem.DEFAULT, true, 3)); + Iterable rows = Collections.singletonList(new Object[] {"1984-04-12 00:00:00"}); + stubExecutorWith(relNode, rows); + + QueryResponse response = executeAndCapture(relNode); + String dump = dumpResponse(response); + + assertEquals(ExprCoreType.DATE, response.getSchema().getColumns().get(0).getExprType(), dump); + assertEquals( + "1984-04-12", response.getResults().get(0).tupleValue().get("d").stringValue(), dump); + } + + /** TimeOnlyType — schema reports TIME, value strips 1970-01-01 prefix. */ + @Test + void executeRelNode_timeOnlyTypeStripsEpochDatePrefix() { + RelNode relNode = + mockRelNodeWithType("t", new TimeOnlyType(RelDataTypeSystem.DEFAULT, true, 3)); + Iterable rows = Collections.singletonList(new Object[] {"1970-01-01 09:00:00"}); + stubExecutorWith(relNode, rows); + + QueryResponse response = executeAndCapture(relNode); + String dump = dumpResponse(response); + + assertEquals(ExprCoreType.TIME, response.getSchema().getColumns().get(0).getExprType(), dump); + assertEquals( + "09:00:00", response.getResults().get(0).tupleValue().get("t").stringValue(), dump); + } + + /** TIME-typed list elements arrive as "1970-01-01[ T]HH:mm:ss[.frac]" — strip the prefix. */ + @Test + void executeRelNode_listOfStringStripsEpochDatePrefix() { + SqlTypeFactoryImpl typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + RelDataType arrayOfVarchar = + typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.VARCHAR), -1); + RelNode relNode = mockRelNodeWithType("time_list", arrayOfVarchar); + java.util.List input = + Arrays.asList( + "1970-01-01 19:36:22", + "1970-01-01T02:05:25", + "1970-01-01 12:34:56.123456789", + "2020-10-13 13:00:00", + "hello"); + stubExecutorWith(relNode, Collections.singletonList(new Object[] {input})); + + QueryResponse response = executeAndCapture(relNode); + String dump = dumpResponse(response); + + java.util.List result = + response.getResults().get(0).tupleValue().get("time_list").collectionValue().stream() + .map(org.opensearch.sql.data.model.ExprValue::stringValue) + .toList(); + assertEquals( + Arrays.asList("19:36:22", "02:05:25", "12:34:56.123456789", "2020-10-13 13:00:00", "hello"), + result, + dump); + } + @Test void executeRelNode_emptyResults() { RelNode relNode = mockRelNode("name", SqlTypeName.VARCHAR); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java index 373881e8543..5533fd90915 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/CoercionUtilsTest.java @@ -17,6 +17,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -51,6 +52,40 @@ public void findCommonWidestType( expectedCommonType, CoercionUtils.resolveCommonType(left, right).orElseGet(() -> null)); } + @Test + void widenArgumentsUnifiesPlainTimestampWithDateUdtBounds() { + // Reproduces the BETWEEN/IN operand set seen on a non-UDT path (e.g. the analytics engine's + // parquet scan): a standard Calcite TIMESTAMP field against EXPR_DATE UDT bounds. + // leastRestrictive + // returns no common type for this mix, but the temporal widening used by comparison operators + // must resolve all three to a single timestamp type so the predicate can run. + RexNode plainTimestampField = + REX_BUILDER.makeInputRef( + OpenSearchTypeFactory.TYPE_FACTORY.createTypeWithNullability( + OpenSearchTypeFactory.TYPE_FACTORY.createSqlType(SqlTypeName.TIMESTAMP), true), + 0); + RexNode dateLower = + REX_BUILDER.makeNullLiteral( + OpenSearchTypeFactory.TYPE_FACTORY.createUDT( + OpenSearchTypeFactory.ExprUDT.EXPR_DATE, true)); + RexNode dateUpper = + REX_BUILDER.makeNullLiteral( + OpenSearchTypeFactory.TYPE_FACTORY.createUDT( + OpenSearchTypeFactory.ExprUDT.EXPR_DATE, true)); + + List widened = + CoercionUtils.widenArguments( + REX_BUILDER, List.of(plainTimestampField, dateLower, dateUpper)); + + assertNotNull(widened); + assertEquals(3, widened.size()); + for (RexNode node : widened) { + assertEquals( + ExprCoreType.TIMESTAMP, + OpenSearchTypeFactory.convertRelDataTypeToExprType(node.getType())); + } + } + @Test void castArgumentsReturnsExactMatchWhenAvailable() { PPLTypeChecker typeChecker = new StubTypeChecker(List.of(List.of(INTEGER), List.of(DOUBLE))); diff --git a/doctest/build.gradle b/doctest/build.gradle index d758a6de4b8..cce64170f56 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -15,6 +15,20 @@ plugins { apply plugin: 'opensearch.testclusters' +configurations { + zipArchive +} + +repositories { + mavenLocal() + maven { url "https://ci.opensearch.org/maven2/" } + mavenCentral() +} + +dependencies { + zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" +} + def path = project(':').projectDir // temporary fix, because currently we are under migration to new architecture. Need to run ./gradlew run from // plugin module, and will only build ppl in it. @@ -156,19 +170,13 @@ check.dependsOn doctest clean.dependsOn(cleanBootstrap) clean.dependsOn(stopPrometheus) -// 2.0.0-alpha1-SNAPSHOT -> 2.0.0.0-alpha1-SNAPSHOT -String opensearch_no_snapshot = opensearch_version.replace('-SNAPSHOT', '') -String[] version_tokens = opensearch_no_snapshot.tokenize('-') -String opensearch_build = version_tokens[0] + '.0' -if (version_tokens.length > 1) { - opensearch_build += '-' + version_tokens[1] +def getJobSchedulerPlugin() { + provider { (RegularFile) (() -> + configurations.zipArchive.asFileTree.matching { + include '**/opensearch-job-scheduler*' + }.singleFile ) + } } -String mlCommonsRemoteFile = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-ml-' + opensearch_build + '.zip' -String mlCommonsPlugin = 'opensearch-ml' - -String bwcOpenSearchJSDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + opensearch_no_snapshot + '/latest/linux/x64/tar/builds/' + - 'opensearch/plugins/opensearch-job-scheduler-' + opensearch_build + '.zip' -String jsPlugin = 'opensearch-job-scheduler' testClusters { docTestCluster { @@ -194,7 +202,7 @@ testClusters { } })) */ - plugin(getJobSchedulerPlugin(jsPlugin, bwcOpenSearchJSDownload)) + plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' } @@ -203,50 +211,3 @@ tasks.register("runRestTestCluster", RunTask) { description = 'Runs OpenSearch SQL plugin' useCluster testClusters.docTestCluster; } - - -def getJobSchedulerPlugin(String jsPlugin, String bwcOpenSearchJSDownload) { - return provider(new Callable() { - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - // Use absolute paths - String basePath = new File('.').getCanonicalPath() - File dir = new File(basePath + File.separator + 'doctest' + File.separator + jsPlugin) - - // Log the directory path for debugging - println("Creating directory: " + dir.getAbsolutePath()) - - // Create directory if it doesn't exist - if (!dir.exists()) { - if (!dir.mkdirs()) { - throw new IOException("Failed to create directory: " + dir.getAbsolutePath()) - } - } - - // Define the file path - File f = new File(dir, jsPlugin + '-' + opensearch_build + '.zip') - - // Download file if it doesn't exist - if (!f.exists()) { - println("Downloading file from: " + bwcOpenSearchJSDownload) - println("Saving to file: " + f.getAbsolutePath()) - - new URL(bwcOpenSearchJSDownload).withInputStream { ins -> - f.withOutputStream { it << ins } - } - } - - // Check if the file was created successfully - if (!f.exists()) { - throw new FileNotFoundException("File was not created: " + f.getAbsolutePath()) - } - - return fileTree(f.getParent()).matching { include f.getName() }.singleFile - } - } - } - }) -} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7d48e415204..db32fcb2c2b 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -525,6 +525,43 @@ testClusters.analyticsEngineSecurityIT { configureSecurityPlugin(testClusters.analyticsEngineSecurityIT) +task analyticsEngineProfileIT(type: RestIntegTestTask) { + dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip, downloadArrowBaseZip, downloadAnalyticsBackendLuceneZip, downloadParquetDataFormatZip, downloadCompositeEngineZip, downloadAnalyticsBackendDatafusionZip + dependsOn ':opensearch-sql-plugin:bundlePlugin' + + systemProperty 'tests.security.manager', 'false' + + filter { + includeTestsMatching 'org.opensearch.sql.analytics.AnalyticsEngineProfileIT' + } +} + +testClusters.analyticsEngineProfileIT { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowBasePlugin()) + plugin(getArrowFlightRpcPlugin()) + plugin(getAnalyticsEnginePlugin()) + plugin(getAnalyticsBackendLucenePlugin()) + plugin(getAnalyticsBackendDatafusionPlugin()) + plugin(getParquetDataFormatPlugin()) + plugin(getCompositeEnginePlugin()) + plugin ":opensearch-sql-plugin" + // Arrow Flight / streaming transport requirements + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' + // Native library path for DataFusion/parquet — pass via -PnativeLibPath=/path/to/release/ + if (project.findProperty('nativeLibPath')) { + systemProperty 'java.library.path', project.findProperty('nativeLibPath') + } +} + task integJdbcTest(type: RestIntegTestTask) { testClusters.findAll {c -> c.clusterName == "integJdbcTest"}.first().with { plugin ":opensearch-sql-plugin" @@ -708,6 +745,9 @@ integTest { exclude 'org/opensearch/sql/doctest/**/*IT.class' exclude 'org/opensearch/sql/correctness/**' + // Exclude tests that require the analytics engine plugin stack (run separately via dedicated tasks) + exclude 'org/opensearch/sql/analytics/AnalyticsEngineProfileIT.class' + // Explain IT is dependent on internal implementation of old engine so it's not necessary // to run these with new engine and not necessary to make this consistent with old engine. exclude 'org/opensearch/sql/legacy/ExplainIT.class' @@ -726,6 +766,12 @@ integTest { // Exclude this IT, because they executed in another task (:integTestWithSecurity) exclude 'org/opensearch/sql/security/**' + // Exclude the analytics-engine smoke test, because it executes in another task + // (:analyticsEngineCompatIT) against a cluster that bundles the analytics-engine plugin + // stack. Running it here against the plain integTest/remoteCluster (no analytics-engine) + // is pointless and leaves it exposed to suite-wide infra flakiness. + exclude 'org/opensearch/sql/plugin/AnalyticsEngineCompatIT.class' + // Workaround for Gradle 9.4.1 ClassCastException in TestEventReporterAsListener.started // (line 58) — the bridge casts a parent test descriptor's reporter to // GroupTestEventReporterInternal but a class-level @Ignore produces a non-composite parent @@ -989,13 +1035,54 @@ task integTestRemote(type: RestIntegTestTask) { systemProperty 'tests.analytics.parquet_indices', System.getProperty("tests.analytics.parquet_indices") } + // Primary-shard count for analytics-backed test indices (default 1). Set to e.g. 3 to + // reproduce multi-shard coordination behavior on a single-node cluster. + if (System.getProperty("tests.analytics.num_shards") != null) { + systemProperty 'tests.analytics.num_shards', System.getProperty("tests.analytics.num_shards") + } + + // True only when the analytics-engine route is active (every test index parquet-backed). Matches + // AnalyticsIndexConfig.isEnabled, which parses the value rather than checking mere presence, so a + // `-Dtests.analytics.parquet_indices=false` run stays on the v2 path. + def analyticsEnabled = Boolean.parseBoolean(System.getProperty("tests.analytics.parquet_indices", "false")) + // Set default query size limit systemProperty 'defaultQuerySizeLimit', '10000' - if (System.getProperty("tests.rest.bwcsuite") == null) { - filter { + filter { + if (System.getProperty("tests.rest.bwcsuite") == null) { excludeTestsMatching "org.opensearch.sql.bwc.*IT" } + + if (analyticsEnabled) { + // Full-text search (search-filter syntax -> Lucene query_string) needs an inverted-index + // reader that parquet-backed analytics indices lack, so it's unsupported on this route. + excludeTestsMatching '*OperatorIT.testEqualOperator' + excludeTestsMatching '*OperatorIT.testNotEqualOperator' + excludeTestsMatching '*OperatorIT.testLessOperator' + excludeTestsMatching '*OperatorIT.testLteOperator' + excludeTestsMatching '*OperatorIT.testGreaterOperator' + excludeTestsMatching '*OperatorIT.testGteOperator' + excludeTestsMatching '*OperatorIT.testNotOperator' + + // === Excludes: PPL ITs doomed by the unsupported-field strip === + // On the AE route TestUtils.AnalyticsIndexConfig strips nested/geo_point/geo_shape/alias + // fields from every test mapping AND its bulk data (the parquet/composite store can't + // scan them). A test whose query references such a field can't pass — the data is gone. + // Verified field-by-field against the index mappings; only doomed work is excluded. + // Gated on the parsed boolean so `-Dtests.analytics.parquet_indices=false` excludes none. + // + // GeoPointFormatsIT: EVERY test reads a geo_point field — whole class doomed. + excludeTestsMatching 'org.opensearch.sql.ppl.GeoPointFormatsIT' + // datatypes_index_mapping: nested_value=nested, geo_point_value=geo_point. + excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_nonnumeric_data_types' + excludeTestsMatching 'org.opensearch.sql.ppl.SystemFunctionIT.typeof_opensearch_types' + // alias_index_mapping: alias_col is type=alias; query is `where alias_col > 1`. + excludeTestsMatching 'org.opensearch.sql.ppl.DataTypeIT.test_alias_data_type' + // CalciteAliasFieldAggregationIT: raw-PUT alias index can't be created on the AE route + // and every test queries alias fields directly — whole class doomed. + excludeTestsMatching 'org.opensearch.sql.calcite.remote.CalciteAliasFieldAggregationIT' + } } // Exclude the same tests that are excluded for integTest diff --git a/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java b/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java new file mode 100644 index 00000000000..4a28115897a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/analytics/AnalyticsEngineProfileIT.java @@ -0,0 +1,218 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.analytics; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +/** + * Integration tests for the analytics engine profile API (profile=true). Verifies that PPL and SQL + * queries with profile=true return stage profiling, and that /_explain returns only the logical + * plan without execution. + * + *

    Runs against the analyticsEngineProfileIT cluster which has the full analytics engine stack. + */ +public class AnalyticsEngineProfileIT extends OpenSearchRestTestCase { + + private static final String INDEX = "profile_test"; + private static boolean initialized = false; + + private void ensureSetup() throws IOException { + if (initialized) return; + enableCalcite(); + createCompositeIndex(); + ingestData(); + initialized = true; + } + + private void enableCalcite() throws IOException { + Request req = new Request("PUT", "/_cluster/settings"); + req.setJsonEntity("{\"persistent\":{\"plugins.calcite.enabled\":true}}"); + client().performRequest(req); + } + + private void createCompositeIndex() throws IOException { + try { + Request req = new Request("PUT", "/" + INDEX); + req.setJsonEntity( + """ + { + "settings": { + "number_of_shards": 2, + "number_of_replicas": 0, + "index.pluggable.dataformat.enabled": true, + "index.pluggable.dataformat": "composite" + }, + "mappings": { + "properties": { + "name": {"type": "keyword"}, + "score": {"type": "double"} + } + } + } + """); + client().performRequest(req); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 400) throw e; + // Index already exists + } + } + + private void ingestData() throws IOException { + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + bulk.setJsonEntity( + String.format( + Locale.ROOT, + """ + {"index":{"_index":"%s"}} + {"name":"alice","score":95.5} + {"index":{"_index":"%s"}} + {"name":"bob","score":87.3} + {"index":{"_index":"%s"}} + {"name":"carol","score":91.0} + """, + INDEX, + INDEX, + INDEX)); + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/x-ndjson"); + bulk.setOptions(opts); + client().performRequest(bulk); + } + + @Test + public void testPplProfileReturnsStages() throws IOException { + ensureSetup(); + JSONObject result = + executeWithProfile("source = " + INDEX + " | stats avg(score) by name", "/_plugins/_ppl"); + + assertTrue("has schema", result.has("schema")); + assertTrue("has datarows", result.has("datarows")); + assertTrue("has profile", result.has("profile")); + + JSONObject profile = result.getJSONObject("profile"); + assertTrue("has query_id", profile.has("query_id")); + assertTrue("has planning_time_ms", profile.has("planning_time_ms")); + assertTrue("has execution_time_ms", profile.has("execution_time_ms")); + assertTrue("has full_plan", profile.has("full_plan")); + + JSONArray stages = profile.getJSONArray("stages"); + assertTrue("at least one stage", stages.length() >= 1); + + JSONObject stage = stages.getJSONObject(0); + assertTrue("stage has stage_id", stage.has("stage_id")); + assertTrue("stage has execution_type", stage.has("execution_type")); + assertTrue("stage has state", stage.has("state")); + assertTrue("stage has elapsed_ms", stage.has("elapsed_ms")); + assertTrue("stage has tasks", stage.has("tasks")); + } + + @Test + public void testSqlProfileReturnsStages() throws IOException { + ensureSetup(); + JSONObject result = executeWithProfile("SELECT * FROM " + INDEX, "/_plugins/_sql"); + + assertTrue("has schema", result.has("schema")); + assertTrue("has datarows", result.has("datarows")); + assertTrue("has profile", result.has("profile")); + + JSONObject profile = result.getJSONObject("profile"); + assertTrue("has query_id", profile.has("query_id")); + assertTrue("has stages", profile.has("stages")); + JSONArray stages = profile.getJSONArray("stages"); + assertTrue("at least one stage", stages.length() >= 1); + } + + @Test + public void testPplProfileStagesShowSucceeded() throws IOException { + ensureSetup(); + JSONObject result = + executeWithProfile("source = " + INDEX + " | fields name, score", "/_plugins/_ppl"); + + JSONObject profile = result.getJSONObject("profile"); + JSONArray stages = profile.getJSONArray("stages"); + + for (int i = 0; i < stages.length(); i++) { + JSONObject stage = stages.getJSONObject(i); + assertEquals("stage succeeded", "SUCCEEDED", stage.getString("state")); + assertTrue("elapsed_ms non-negative", stage.getLong("elapsed_ms") >= 0); + } + } + + @Test + public void testPplProfileTasksHaveNodeAndTiming() throws IOException { + ensureSetup(); + JSONObject result = + executeWithProfile("source = " + INDEX + " | fields name", "/_plugins/_ppl"); + + JSONObject profile = result.getJSONObject("profile"); + JSONArray stages = profile.getJSONArray("stages"); + + boolean foundTasks = false; + for (int i = 0; i < stages.length(); i++) { + JSONArray tasks = stages.getJSONObject(i).getJSONArray("tasks"); + if (tasks.length() > 0) { + foundTasks = true; + JSONObject task = tasks.getJSONObject(0); + assertTrue("task has node", task.has("node")); + assertTrue("task has state", task.has("state")); + assertTrue("task has elapsed_ms", task.has("elapsed_ms")); + } + } + assertTrue("at least one stage has tasks", foundTasks); + } + + @Test + public void testPplExplainReturnsOnlyPlan() throws IOException { + ensureSetup(); + Request request = new Request("POST", "/_plugins/_ppl/_explain"); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"source = %s | fields name, score\"}", INDEX)); + Response response = client().performRequest(request); + JSONObject result = new JSONObject(entityAsString(response)); + + assertTrue("has calcite", result.has("calcite")); + JSONObject calcite = result.getJSONObject("calcite"); + assertTrue("has logical", calcite.has("logical")); + assertFalse("no profile in explain", calcite.has("profile")); + } + + @Test + public void testSqlExplainReturnsOnlyPlan() throws IOException { + ensureSetup(); + Request request = new Request("POST", "/_plugins/_sql/_explain"); + request.setJsonEntity(String.format(Locale.ROOT, "{\"query\": \"SELECT * FROM %s\"}", INDEX)); + Response response = client().performRequest(request); + JSONObject result = new JSONObject(entityAsString(response)); + + assertTrue("has calcite", result.has("calcite")); + JSONObject calcite = result.getJSONObject("calcite"); + assertTrue("has logical", calcite.has("logical")); + assertFalse("no profile in explain", calcite.has("profile")); + } + + private JSONObject executeWithProfile(String query, String endpoint) throws IOException { + Request request = new Request("POST", endpoint); + request.setJsonEntity( + String.format(Locale.ROOT, "{\"query\": \"%s\", \"profile\": true}", query)); + Response response = client().performRequest(request); + return new JSONObject(entityAsString(response)); + } + + private static String entityAsString(Response response) throws IOException { + return new String( + response.getEntity().getContent().readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java new file mode 100644 index 00000000000..a194b2999e2 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/AnalyticsUnsupportedFieldStripVerifyIT.java @@ -0,0 +1,434 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.legacy.TestUtils.analyticsDroppedFields; +import static org.opensearch.sql.legacy.TestUtils.getResourceFilePath; +import static org.opensearch.sql.legacy.TestUtils.getResponseBody; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Assume; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Exhaustive, fail-loud proof that data ingestion on the analytics-engine route is free of + * unsupported-field problems. For EVERY {@link Index} value it forces a clean (re)load through the + * real {@link #loadIndex} harness path against the AE cluster, then verifies, per dataset: + * + *

      + *
    1. The real load path ran from a clean index — the index is deleted first, so {@code + * loadIndex}'s {@code if (!isIndexExist)} guard cannot short-circuit and inspect a stale + * index left by an earlier test. {@code loadDataByRestClient} now fails loudly on bulk {@code + * errors=true}, so a partial ingestion surfaces here as a load failure, not silently. + *
    2. Mapping is clean — the live mapping pulled back from the cluster contains none of + * the unsupported types at any depth. + *
    3. Source agrees with mapping — no sampled {@code _source} doc still carries a stripped + * path (a leftover value would dynamic-map the field back, re-introducing the unsupported + * type). + *
    4. Doc count is sane — a fixture with N source docs ingests exactly N (proving no item + * silently dropped). Skipped only for a degenerate index whose strip left no scannable + * columns (e.g. cascaded_nested, all-{@code nested}), which the engine can't query at all. + *
    + * + *

    This converts the "silent failure" risk (a missed strip surfacing only as an unrelated {@code + * expected:<1> but was:<0>} assertion in some downstream IT) into a direct, attributable failure on + * the exact index and field. Only runs on the AE route ({@code -Dtests.analytics.parquet_indices}); + * assume-skipped otherwise. + */ +public class AnalyticsUnsupportedFieldStripVerifyIT extends PPLIntegTestCase { + + private static final Logger LOG = + LogManager.getLogger(AnalyticsUnsupportedFieldStripVerifyIT.class); + + // Single source of truth — the same set the load-path strip uses + // (TestUtils.AnalyticsIndexConfig). Importing it (not re-listing) keeps verifier and stripper in + // lockstep. + private static final Set UNSUPPORTED = + org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig.UNSUPPORTED_FIELD_TYPES; + + /** + * Field types the parquet/composite store also rejects but that are out of scope for the strip + * (by product decision). An index whose creation fails solely because of one of these is skipped + * — but only after confirming its mapping carries no {@link #UNSUPPORTED} type (see {@link + * #safeToSkipForOutOfScopeType}), so a mixed failure can't be hidden. + */ + private static final Set OUT_OF_SCOPE_TYPES = Set.of("join"); + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + } + + @Test + public void everyDatasetIngestsCleanlyOnAnalyticsEngine() throws IOException { + Assume.assumeTrue( + "AE-route only: requires -Dtests.analytics.parquet_indices=true", + isAnalyticsParquetIndicesEnabled()); + + List failures = new ArrayList<>(); + for (Index index : Index.values()) { + String name = index.getName(); + String mapping = index.getMapping(); + + // Force the real load path to run: delete any index a prior test left behind so loadIndex's + // `if (!isIndexExist)` guard can't short-circuit and inspect a stale index. + try { + deleteIndexIfExists(name); + } catch (IOException e) { + failures.add("[" + index.name() + " -> " + name + "] could not delete: " + rootMessage(e)); + continue; + } + + try { + loadIndex(index); + } catch (Exception e) { + if (isMissingDatasetFile(e)) { + // Dangling enum entry with no dataset file on disk — pre-existing repo issue, not an + // unsupported-field problem. Skip. + continue; + } + if (safeToSkipForOutOfScopeType(e, mapping)) { + // Creation failed solely on an out-of-scope type (e.g. join) AND the mapping carries no + // unsupported type we're responsible for — not our concern. Skip. + continue; + } + failures.add( + "[" + + index.name() + + " -> " + + name + + "] loadIndex FAILED (ingestion error): " + + rootMessage(e)); + continue; + } + + // Leg 2: live mapping carries no unsupported type at any depth. + List offending = unsupportedFieldsInLiveMapping(name); + if (!offending.isEmpty()) { + failures.add( + "[" + + index.name() + + " -> " + + name + + "] live mapping still has unsupported fields: " + + offending); + } + + // Leg 3: no sampled _source doc still carries a stripped path. + Set> dropped = analyticsDroppedFields(mapping); + List leftover = strippedPathsStillInSource(name, dropped); + if (!leftover.isEmpty()) { + failures.add( + "[" + + index.name() + + " -> " + + name + + "] _source still carries stripped paths " + + leftover + + " (dynamic mapping would re-introduce the unsupported type)"); + } + + // Leg 4: a non-empty fixture ingests exactly its source-doc count — proving no item silently + // dropped. Skipped for a degenerate index whose strip left no scannable columns (can't + // query). + // The count goes through PPL `stats count()`; if that query errors for a cluster-side reason + // unrelated to the strip (e.g. it transiently touches a parquet-backed system index), we + // record a transparent skip rather than fail — Legs 1-3 plus the load-time bulk errors=false + // check (TestUtils.loadDataByRestClient) already catch partial ingestion directly. + int expectedDocs = sourceDocCount(index); + if (expectedDocs > 0 && liveMappingHasColumns(name)) { + Integer actualDocs = liveDocCountOrNull(name); + if (actualDocs == null) { + skippedDocCount.add(index.name() + " (count query errored — see Legs 1-3 + bulk check)"); + } else if (actualDocs != expectedDocs) { + failures.add( + "[" + + index.name() + + " -> " + + name + + "] ingested " + + actualDocs + + " docs but dataset" + + " has " + + expectedDocs + + " (partial/failed ingestion)"); + } + } + } + + // Surface skipped doc-count checks so coverage gaps are visible, not silent. + if (!skippedDocCount.isEmpty()) { + LOG.info( + "Leg 4 (doc-count) skipped for {} dataset(s): {}", + skippedDocCount.size(), + String.join(", ", skippedDocCount)); + } + + if (!failures.isEmpty()) { + throw new AssertionError( + "Unsupported-field ingestion problems on the AE route (" + + failures.size() + + "):\n " + + String.join("\n ", failures)); + } + } + + private final List skippedDocCount = new ArrayList<>(); + + private void deleteIndexIfExists(String indexName) throws IOException { + try { + client().performRequest(new Request("DELETE", "/" + indexName)); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 404) { + throw e; + } + } + } + + /** Pull the live mapping back from the cluster and collect any unsupported-typed field paths. */ + private List unsupportedFieldsInLiveMapping(String indexName) throws IOException { + List offending = new ArrayList<>(); + JSONObject props = liveMappingProperties(indexName); + if (props != null) { + collectUnsupported(props, "", offending); + } + return offending; + } + + /** + * True when the index's live mapping still has at least one top-level property. An index whose + * mapping was reduced to nothing by the strip (all-unsupported dataset, e.g. cascaded_nested) has + * no scannable columns and can't be queried on the AE route — Leg 4 is skipped for it. + */ + private boolean liveMappingHasColumns(String indexName) throws IOException { + JSONObject props = liveMappingProperties(indexName); + return props != null && !props.isEmpty(); + } + + /** {@code mappings.properties} of the live index, or null when the mapping has no properties. */ + private JSONObject liveMappingProperties(String indexName) throws IOException { + Response response = client().performRequest(new Request("GET", "/" + indexName + "/_mapping")); + JSONObject mappings = + new JSONObject(getResponseBody(response)) + .getJSONObject(indexName) + .getJSONObject("mappings"); + return mappings.has("properties") ? mappings.getJSONObject("properties") : null; + } + + /** + * Sample up to a few docs via the search API and report any {@code dropped} path that still + * exists in a returned {@code _source}. A leftover value is what would let dynamic mapping + * re-create the unsupported field after refresh. + */ + private List strippedPathsStillInSource(String indexName, Set> dropped) + throws IOException { + List leftover = new ArrayList<>(); + if (dropped.isEmpty()) { + return leftover; + } + Request search = new Request("POST", "/" + indexName + "/_search?size=50"); + Response response; + try { + response = client().performRequest(search); + } catch (ResponseException e) { + // A degenerate index (no columns) can't be searched; nothing to check. + return leftover; + } + JSONObject body = new JSONObject(getResponseBody(response)); + JSONArray hits = body.getJSONObject("hits").getJSONArray("hits"); + for (int i = 0; i < hits.length(); i++) { + JSONObject source = hits.getJSONObject(i).optJSONObject("_source"); + if (source == null) { + continue; + } + for (List path : dropped) { + if (pathPresent(source, path, 0)) { + String dotted = String.join(".", path); + if (!leftover.contains(dotted)) { + leftover.add(dotted); + } + } + } + } + return leftover; + } + + /** Mirror of TestUtils.removePath traversal: is {@code path[idx..]} present in {@code node}? */ + private boolean pathPresent(Object node, List path, int idx) { + String part = path.get(idx); + boolean last = idx == path.size() - 1; + if (node instanceof JSONObject) { + JSONObject obj = (JSONObject) node; + if (!obj.has(part)) { + return false; + } + return last || pathPresent(obj.get(part), path, idx + 1); + } else if (node instanceof JSONArray) { + JSONArray arr = (JSONArray) node; + for (int j = 0; j < arr.length(); j++) { + if (pathPresent(arr.get(j), path, idx)) { + return true; + } + } + } + return false; + } + + /** + * Live doc count of the index, obtained through the analytics-engine query path ({@code + * source=idx | stats count()}). The REST {@code _count} API is NOT usable here — on a + * parquet-backed {@code DataFormatAwareEngine} index it fails with "Cannot apply function ... + * directly on IndexShard". Counting via PPL also exercises the real scan path, so the result + * proves the data both ingested and is readable on the AE route. + */ + private Integer liveDocCountOrNull(String indexName) { + String query = String.format("source=%s | stats count() as c", indexName); + try { + JSONObject result = executeQuery(query); + return result.getJSONArray("datarows").getJSONArray(0).getInt(0); + } catch (Exception e) { + // The count query can fail for cluster-side reasons unrelated to the strip (e.g. it touches a + // parquet-backed system index that can't serve a shard-level count). Don't let that mask the + // strip proof — return null so the caller records a transparent skip. + LOG.info("count query failed for [{}]: {}", indexName, rootMessage(e)); + return null; + } + } + + /** + * Number of source (non-action) lines in a dataset's bulk NDJSON file — i.e. how many docs the + * load is expected to ingest. Returns 0 when the index has no dataset file (mapping-only enum + * entries), which makes Leg 4 a no-op for those. + */ + private static int sourceDocCount(Index index) throws IOException { + String dataSet = index.getDataSet(); + if (dataSet == null || dataSet.isEmpty()) { + return 0; + } + String body = new String(Files.readAllBytes(Paths.get(getResourceFilePath(dataSet)))); + int docs = 0; + for (String line : body.split("\n", -1)) { + String trimmed = line.trim(); + if (trimmed.isEmpty() || trimmed.charAt(0) != '{') { + continue; + } + JSONObject obj = new JSONObject(trimmed); + boolean isActionLine = + obj.has("index") || obj.has("create") || obj.has("update") || obj.has("delete"); + if (!isActionLine) { + docs++; + } + } + return docs; + } + + private void collectUnsupported(JSONObject properties, String prefix, List out) { + for (String field : properties.keySet()) { + JSONObject def = properties.optJSONObject(field); + if (def == null) { + continue; + } + String type = def.optString("type", null); + String path = prefix.isEmpty() ? field : prefix + "." + field; + if (type != null && UNSUPPORTED.contains(type)) { + out.add(path + ":" + type); + } + if (def.has("properties")) { + collectUnsupported(def.getJSONObject("properties"), path, out); + } + } + } + + /** + * Safe to skip an index whose creation failed, only when BOTH hold: (a) the error names an + * out-of-scope type (e.g. join), AND (b) the dataset's mapping contains no {@link #UNSUPPORTED} + * type. The second condition prevents a mixed failure (a join issue masking a missed unsupported + * field) from being silently skipped — if any unsupported type is present, we do NOT skip. + */ + private static boolean safeToSkipForOutOfScopeType(Throwable t, String mapping) { + String msg = rootMessage(t); + if (msg == null) { + return false; + } + boolean namesOutOfScope = false; + for (String type : OUT_OF_SCOPE_TYPES) { + if (msg.contains("type: " + type) || msg.contains("type [" + type + "]")) { + namesOutOfScope = true; + break; + } + } + if (!namesOutOfScope) { + return false; + } + return !mappingContainsUnsupportedType(mapping); + } + + /** True if the raw mapping JSON declares any {@link #UNSUPPORTED} field type at any depth. */ + private static boolean mappingContainsUnsupportedType(String mapping) { + if (mapping == null || mapping.isEmpty()) { + return false; + } + JSONObject root = new JSONObject(mapping); + JSONObject mappings = root.optJSONObject("mappings"); + if (mappings == null || !mappings.has("properties")) { + return false; + } + return propertiesContainUnsupported(mappings.getJSONObject("properties")); + } + + private static boolean propertiesContainUnsupported(JSONObject properties) { + for (String field : properties.keySet()) { + JSONObject def = properties.optJSONObject(field); + if (def == null) { + continue; + } + String type = def.optString("type", null); + if (type != null && UNSUPPORTED.contains(type)) { + return true; + } + if (def.has("properties") && propertiesContainUnsupported(def.getJSONObject("properties"))) { + return true; + } + } + return false; + } + + /** True when the failure is a missing dataset file on disk (pre-existing dangling enum entry). */ + private static boolean isMissingDatasetFile(Throwable t) { + for (Throwable r = t; r != null && r.getCause() != r; r = r.getCause()) { + if (r instanceof java.nio.file.NoSuchFileException + || r instanceof java.io.FileNotFoundException) { + return true; + } + if (r.getCause() == null) { + break; + } + } + return false; + } + + private static String rootMessage(Throwable t) { + Throwable r = t; + while (r.getCause() != null && r.getCause() != r) { + r = r.getCause(); + } + return r.getMessage(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java index dfe6ba17b62..5a23492ca6e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -31,6 +31,9 @@ public class CalciteAliasFieldAggregationIT extends PPLIntegTestCase { @Override public void init() throws Exception { super.init(); + // Excluded on the analytics-engine route from integ-test/build.gradle (alias fields are + // unsupported there — the raw-PUT index can't even be created and these tests query the alias + // fields directly), alongside the other AE-route exclusions kept in one place. enableCalcite(); createTestIndexWithAliasFields(); loadIndex(Index.DATA_TYPE_ALIAS); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java index 8bcfd034a8d..3fb8219fe12 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java @@ -85,10 +85,12 @@ public void testBinWithMinspan() throws IOException { public void testBinBasicFunctionality() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | bin age span=5 | fields age | head 3", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | bin age span=5 | sort account_number | fields age | head 3", + TEST_INDEX_ACCOUNT)); verifySchema(result, schema("age", null, "string")); - verifyDataRows(result, rows("30-35"), rows("35-40"), rows("25-30")); + verifyDataRows(result, rows("25-30"), rows("30-35"), rows("20-25")); } @Test @@ -195,7 +197,9 @@ public void testBinOnlyWithoutAggregation() throws IOException { JSONObject binOnlyResult = executeQuery( String.format( - "source=%s" + " | bin @timestamp span=4h" + " | fields `@timestamp` | head 3", + "source=%s" + + " | bin @timestamp span=4h" + + " | fields `@timestamp` | sort `@timestamp` | head 3", TEST_INDEX_TIME_DATA)); // Verify schema and that binning works correctly @@ -235,7 +239,7 @@ public void testBinWithMonthlySpan() throws IOException { executeQuery( String.format( "source=%s | bin @timestamp span=4mon as cate | fields" - + " cate, @timestamp | head 5", + + " cate, @timestamp | sort @timestamp | head 5", TEST_INDEX_TIME_DATA)); verifySchema(result, schema("cate", null, "string"), schema("@timestamp", null, "timestamp")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java index f51ffabdc35..d2813f6eb17 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteErrorReportStageIT.java @@ -11,6 +11,7 @@ import java.io.IOException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; import org.opensearch.client.ResponseException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -214,4 +215,36 @@ public void testStageDescriptionIsUserFriendly() throws IOException { || stageDescription.toLowerCase().contains("run") || stageDescription.toLowerCase().contains("query")); } + + // An alias field whose path targets a text multi-field (e.g. "source.keyword") is not present in + // the flattened mapping. It used to surface an opaque NullPointerException; it must now report a + // structured FIELD_NOT_FOUND error with a suggestion. + @Test + public void testAliasToUnresolvablePathIncludesStructuredError() throws IOException { + String index = "test_alias_unresolved_keyword"; + Request createIndex = new Request("PUT", "/" + index); + createIndex.setJsonEntity( + "{ \"mappings\": { \"properties\": {" + + " \"source\": { \"type\": \"text\", \"fields\": { \"keyword\": { \"type\":" + + " \"keyword\" } } }," + + " \"source_alias\": { \"type\": \"alias\", \"path\": \"source.keyword\" } } } }"); + client().performRequest(createIndex); + + ResponseException exception = + assertThrows(ResponseException.class, () -> executeQuery("source=" + index)); + + JSONObject error = + new JSONObject(getResponseBody(exception.getResponse())).getJSONObject("error"); + + assertEquals("FIELD_NOT_FOUND", error.getString("code")); + assertTrue( + "Details should name the alias field and path", + error + .getString("details") + .contains("Alias field [source_alias] refers to unresolved path [source.keyword]")); + JSONObject context = error.getJSONObject("context"); + assertEquals("source_alias", context.getString("alias_field")); + assertEquals("source.keyword", context.getString("alias_path")); + assertTrue("Should include a suggestion", error.has("suggestion")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java index f7e994223c6..a7aac6765b5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ARRAY; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -295,6 +296,10 @@ public void testExpandWithEval() throws Exception { @Test public void testExpandEmptyArray() throws Exception { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 6; Request insertRequest = new Request( @@ -324,6 +329,10 @@ public void testExpandEmptyArray() throws Exception { @Test public void testExpandOnNullField() throws Exception { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 6; Request insertRequest = new Request( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 8ad0be5cc88..d25631c5ace 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2153,6 +2153,29 @@ public void testTransposeExplain() throws IOException { + "| transpose 4 column_name='column_names'")); } + /** + * With a user {@code where} clause preceding {@code dedup}, the physical plan must push both the + * filter and the dedup-as-aggregation into the OpenSearch scan, not fall back to an in-memory + * {@code ROW_NUMBER} window above a row-fetching scan. + */ + @Test + public void testDedupAfterWherePushDown() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String result = + explainQueryToString( + "source=opensearch-sql_test_index_account | where age > 25 | dedup gender"); + assertTrue( + "Expected user where filter pushed down to the scan:\n" + result, + result.contains("FILTER->>($8, 25)")); + assertTrue( + "Expected dedup pushed down as AGGREGATION (composite + top_hits):\n" + result, + result.contains("AGGREGATION->")); + assertFalse( + "Unexpected EnumerableWindow — dedup fell back to the in-memory ROW_NUMBER form:\n" + + result, + result.contains("EnumerableWindow")); + } + public void testComplexDedup() throws IOException { enabledOnlyWhenPushdownIsEnabled(); String expected = loadExpectedPlan("explain_dedup_complex1.yaml"); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java index 9afb63056e8..546adc4755b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMVAppendFunctionIT.java @@ -107,8 +107,8 @@ public void testMvappendWithRealFields() throws IOException { executeQuery( source( TEST_INDEX_BANK, - "eval result = mvappend(firstname, lastname) | head 1 | fields firstname, lastname," - + " result")); + "eval result = mvappend(firstname, lastname) | sort account_number | head 1 |" + + " fields firstname, lastname, result")); verifySchema( actual, @@ -179,7 +179,7 @@ public void testMvappendInWhereClause() throws IOException { source( TEST_INDEX_BANK, "eval combined = mvappend(firstname, lastname) | where array_length(combined) = 2 |" - + " head 1 | fields firstname, lastname, combined")); + + " sort account_number | head 1 | fields firstname, lastname, combined")); verifySchema( actual, @@ -198,8 +198,8 @@ public void testMvappendWithComplexExpression() throws IOException { executeQuery( source( TEST_INDEX_BANK, - "eval result = mvappend(array(age), array(age * 2), age + 10) | head 1 | fields" - + " age, result")); + "eval result = mvappend(array(age), array(age * 2), age + 10) | sort" + + " account_number | head 1 | fields age, result")); verifySchema(actual, schema("age", "int"), schema("result", "array")); verifyDataRows(actual, rows(32, List.of(32, 64, 42))); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java index d0a4882fb1f..a738132744b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultisearchCommandIT.java @@ -325,8 +325,10 @@ public void testMultisearchCrossIndexFieldSelection() throws IOException { executeQuery( String.format( "| multisearch " - + "[search source=%s | fields firstname, balance | head 2] " - + "[search source=%s | fields description, place_id | head 2]", + + "[search source=%s | sort account_number | fields firstname, balance" + + " | head 2] " + + "[search source=%s | sort place_id | fields description, place_id" + + " | head 2]", TEST_INDEX_ACCOUNT, TEST_INDEX_LOCATIONS_TYPE_CONFLICT)); verifySchema( @@ -338,8 +340,8 @@ public void testMultisearchCrossIndexFieldSelection() throws IOException { verifyDataRows( result, + rows("Bradshaw", 16623L, null, null), rows("Amber", 39225L, null, null), - rows("Hattie", 5686L, null, null), rows(null, null, "Central Park", 1001), rows(null, null, "Times Square", 1002)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index ec80e27ba5a..a5937d06f31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -25,7 +25,6 @@ import java.util.List; import org.json.JSONObject; import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -52,16 +51,11 @@ public void init() throws Exception { @Test public void testSimpleCount0() throws IOException { - Request request1 = new Request("PUT", "/test/_doc/1?refresh=true"); - request1.setJsonEntity("{\"name\": \"hello\", \"age\": 20}"); - client().performRequest(request1); - Request request2 = new Request("PUT", "/test/_doc/2?refresh=true"); - request2.setJsonEntity("{\"name\": \"world\", \"age\": 30}"); - client().performRequest(request2); - - JSONObject actual = executeQuery("source=test | stats count() as c"); + // A bare auto-created index isn't parquet-backed; use the parquet-aware bank index (7 docs). + JSONObject actual = + executeQuery(String.format("source=%s | stats count() as c", TEST_INDEX_BANK)); verifySchema(actual, schema("c", "bigint")); - verifyDataRows(actual, rows(2)); + verifyDataRows(actual, rows(7)); } @Test @@ -548,8 +542,8 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 15minute) as" - + " datetime_span", + "source=%s | sort key | head 5 | stats count(datetime0) by span(datetime0," + + " 15minute) as datetime_span", TEST_INDEX_CALCS)); verifySchema( actual, schema("datetime_span", "timestamp"), schema("count(datetime0)", "bigint")); @@ -564,8 +558,8 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 5second) as" - + " datetime_span", + "source=%s | sort key | head 5 | stats count(datetime0) by span(datetime0," + + " 5second) as datetime_span", TEST_INDEX_CALCS)); verifySchema( actual, schema("datetime_span", "timestamp"), schema("count(datetime0)", "bigint")); @@ -580,8 +574,8 @@ public void testCountByCustomTimeSpanWithDifferentUnits() throws IOException { actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0) by span(datetime0, 3month) as" - + " datetime_span", + "source=%s | sort key | head 5 | stats count(datetime0) by span(datetime0," + + " 3month) as datetime_span", TEST_INDEX_CALCS)); verifySchema( actual, schema("datetime_span", "timestamp"), schema("count(datetime0)", "bigint")); @@ -593,8 +587,8 @@ public void testCountByNullableTimeSpan() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0), count(datetime1) by span(time1," - + " 15minute) as time_span", + "source=%s | sort key | head 5 | stats count(datetime0), count(datetime1) by" + + " span(time1, 15minute) as time_span", TEST_INDEX_CALCS)); verifySchema( actual, @@ -980,7 +974,11 @@ public void testPercentile() throws IOException { "source=%s | stats percentile(balance, 50) as p50, percentile(balance, 90) as p90", TEST_INDEX_BANK)); verifySchema(actual, schema("p50", "bigint"), schema("p90", "bigint")); - verifyDataRows(actual, rows(32838, 48086)); + // percentile() is approximate. The analytics-engine backend (DataFusion) uses a different + // t-digest interpolation than the Calcite/OpenSearch percentile_approx implementation, so p90 + // lands on a different value (p50 agrees). Both are valid approximations. + int expectedP90 = isAnalyticsParquetIndicesEnabled() ? 46576 : 48086; + verifyDataRows(actual, rows(32838, expectedP90)); } @Test @@ -990,14 +988,18 @@ public void testSumGroupByNullValue() throws IOException { String.format( "source=%s | stats sum(balance) as a by age", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(response, schema("a", null, "bigint"), schema("age", null, "int")); + // SUM of an all-null bucket is null per the SQL spec. The DSL-pushdown path returns 0 instead + // (a known pushdown quirk); the analytics-engine backend (DataFusion) follows the spec like + // Calcite-no-pushdown and returns null. See testSumNull and #3408. + Object emptySum = (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) ? null : 0; verifyDataRows( response, - rows(isPushdownDisabled() ? null : 0, null), + rows(emptySum, null), rows(32838, 28), rows(39225, 32), rows(4180, 33), rows(48086, 34), - rows(isPushdownDisabled() ? null : 0, 36)); + rows(emptySum, 36)); } @Test @@ -1061,7 +1063,9 @@ public void testSumNull() throws IOException { + " ],\n" + " \"datarows\": [\n" + " [\n" - + (isPushdownDisabled() ? " null\n" : " 0\n") + + ((isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) + ? " null\n" + : " 0\n") + " ]\n" + " ],\n" + " \"total\": 1,\n" diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java index 4c2c8176690..bf16324e604 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLBasicIT.java @@ -500,7 +500,10 @@ public void testNotBetween3() throws IOException { verifyDataRows(actual, rows("Hattie", 36), rows("Elinor", 36)); } + @Test public void testDateBetween() throws IOException { + // birthdate is a TIMESTAMP-typed field; the bounds are DATE literals. BETWEEN must coerce the + // mixed temporal operands rather than reject them with "expression types are incompatible". JSONObject actual = executeQuery( String.format( @@ -512,6 +515,74 @@ public void testDateBetween() throws IOException { actual, rows("Nanette", "2018-06-23 00:00:00"), rows("Elinor", "2018-06-27 00:00:00")); } + /** + * A timestamp range comparison AND'd with an {@code IN} clause must push down and return rows. + */ + @Test + public void testTimestampRangeWithInClausePushDown() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | where birthdate > timestamp('2018-06-01 00:00:00') | where state in" + + " ('IL', 'TN', 'WA') | fields firstname, state, birthdate", + TEST_INDEX_BANK)); + verifySchema( + actual, + schema("firstname", "string"), + schema("state", "string"), + schema("birthdate", "timestamp")); + verifyDataRows(actual, rows("Elinor", "WA", "2018-06-27 00:00:00")); + } + + @Test + public void testDateIn() throws IOException { + // birthdate is a TIMESTAMP-typed field; the IN values are DATE literals. visitIn must compare + // each value in the field's temporal domain (via PPL `=`) rather than letting leastRestrictive + // collapse the common type to VARCHAR and string-compare mismatched renderings — which silently + // matched nothing without pushdown. See visitIn in CalciteRexNodeVisitor. + JSONObject actual = + executeQuery( + String.format( + "source=%s | where birthdate in (DATE '2018-06-23', DATE '2018-06-27') |" + + " fields firstname, birthdate", + TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string"), schema("birthdate", "timestamp")); + verifyDataRows( + actual, rows("Nanette", "2018-06-23 00:00:00"), rows("Elinor", "2018-06-27 00:00:00")); + } + + @Test + public void testDateNotIn() throws IOException { + // Complement of testDateIn: NOT IN over a temporal field. Exercises the complemented-points + // pushdown branch, which must also format timestamp values rather than emit a flat terms query. + JSONObject actual = + executeQuery( + String.format( + "source=%s | where birthdate not in (DATE '2018-06-23', DATE '2018-06-27') |" + + " fields firstname | sort firstname", + TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string")); + verifyDataRows( + actual, + rows("Amber JOHnny"), + rows("Dale"), + rows("Dillard"), + rows("Hattie"), + rows("Virginia")); + } + + @Test + public void testIn() throws IOException { + // Non-temporal IN keeps the leastRestrictive + makeIn path; guards against the temporal-field + // special-case in visitIn regressing membership tests over ordinary columns. + JSONObject actual = + executeQuery( + String.format( + "source=%s | where age in (32, 36) | fields firstname, age", TEST_INDEX_BANK)); + verifySchema(actual, schema("firstname", "string"), schema("age", "int")); + verifyDataRows(actual, rows("Amber JOHnny", 32), rows("Hattie", 36), rows("Elinor", 36)); + } + @Test public void testXor() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java index b7e16d1da8b..84159c1bb96 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLCaseFunctionIT.java @@ -25,6 +25,7 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; public class CalcitePPLCaseFunctionIT extends PPLIntegTestCase { + @Override public void init() throws Exception { super.init(); @@ -48,14 +49,15 @@ private void appendDataForBadResponse() throws IOException { new Request("PUT", "/" + TestsConstants.TEST_INDEX_WEBLOGS + "/_doc/8?refresh=true"); request2.setJsonEntity( "{\"host\": \"0.0.0.2\", \"method\": \"GET\", \"url\":" - + " \"/shuttle/missions/sts-73/mission-sts-73.html\", \"response\": \"500\", \"bytes\":" - + " \"4085\"}"); + + " \"/shuttle/missions/sts-73/mission-sts-73.html\", \"response\": \"500\"," + + " \"bytes\": \"4085\"}"); client().performRequest(request2); Request request3 = new Request("PUT", "/" + TestsConstants.TEST_INDEX_WEBLOGS + "/_doc/9?refresh=true"); request3.setJsonEntity( - "{\"host\": \"::3\", \"method\": \"GET\", \"url\": \"/shuttle/countdown/countdown.html\"," - + " \"response\": \"403\", \"bytes\": \"3985\"}"); + "{\"host\": \"::3\", \"method\": \"GET\", \"url\":" + + " \"/shuttle/countdown/countdown.html\", \"response\": \"403\", \"bytes\":" + + " \"3985\"}"); client().performRequest(request3); Request request4 = new Request("PUT", "/" + TestsConstants.TEST_INDEX_WEBLOGS + "/_doc/10?refresh=true"); @@ -308,7 +310,7 @@ public void testCaseCanBePushedDownAsRangeQuery() throws IOException { TEST_INDEX_BANK)); verifySchema(actual4, schema("avg(balance)", "double"), schema("age_range", "string")); // There's such a discrepancy because null cannot be the key for a range query - if (isPushdownDisabled()) { + if (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) { verifyDataRows( actual4, rows(32838.0, "u30"), @@ -463,7 +465,7 @@ public void testCaseAggWithNullValues() throws IOException { TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("avg(age)", "double"), schema("age_category", "string")); // There is such discrepancy because range aggregations will ignore null values - if (isPushdownDisabled()) { + if (isPushdownDisabled() || isAnalyticsParquetIndicesEnabled()) { verifyDataRows( actual, rows(10, "teenager"), diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java index fd9a5cff774..a0e6c2679fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLEnhancedCoalesceIT.java @@ -73,8 +73,8 @@ public void testCoalesceWithLiterals() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(name, 123, 'unknown') | fields name, result |" - + " head 1", + "source=%s | eval result = coalesce(name, 123, 'unknown') | sort - age | fields" + + " name, result | head 1", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -100,8 +100,8 @@ public void testCoalesceWithMultipleFields() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(name, age, year, month) | fields name, age," - + " year, month, result | head 2", + "source=%s | eval result = coalesce(name, age, year, month) | sort - age | fields" + + " name, age, year, month, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema( @@ -139,8 +139,8 @@ public void testCoalesceWithNonExistentField() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(nonexistent_field, name) | fields name, result" - + " | head 2", + "source=%s | eval result = coalesce(nonexistent_field, name) | sort - age" + + " | fields name, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -234,7 +234,8 @@ public void testCoalesceWithEmptyString() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce('', name) | fields name, result | head 1", + "source=%s | eval result = coalesce('', name) | sort - age | fields name, result" + + " | head 1", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -247,7 +248,8 @@ public void testCoalesceWithSpaceString() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(' ', name) | fields name, result | head 1", + "source=%s | eval result = coalesce(' ', name) | sort - age | fields name, result" + + " | head 1", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("name", "string"), schema("result", "string")); @@ -274,8 +276,8 @@ public void testCoalesceWithCompatibleNumericTypes() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(age, year, 999) | fields age, year, result |" - + " head 2", + "source=%s | eval result = coalesce(age, year, 999) | sort - age | fields age," + + " year, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("age", "int"), schema("year", "int"), schema("result", "int")); @@ -288,7 +290,7 @@ public void testCoalesceTypeCoercionWithMixedTypes() throws IOException { executeQuery( String.format( "source=%s | eval result = coalesce(nonexistent_field, age," - + " 'default') | fields age, result | head 2", + + " 'default') | sort - age | fields age, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema(actual, schema("age", "int"), schema("result", "string")); @@ -300,8 +302,8 @@ public void testCoalesceWithCompatibleNumericAndTemporalTypes() throws IOExcepti JSONObject actual = executeQuery( String.format( - "source=%s | eval result = coalesce(age, year, month) | fields age, year, month," - + " result | head 2", + "source=%s | eval result = coalesce(age, year, month) | sort - age | fields age," + + " year, month, result | head 2", TEST_INDEX_STATE_COUNTRY_WITH_NULL)); verifySchema( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java index 72510e589a9..716eead0478 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLGrokIT.java @@ -34,7 +34,8 @@ public void testGrokEmail() throws IOException { executeQuery( String.format( Locale.ROOT, - "source = %s | grok email '%s' | head 3 | fields email, host", + "source = %s | grok email '%s' | sort account_number | head 3 | fields email," + + " host", TEST_INDEX_BANK, ".+@%{HOSTNAME:host}")); verifySchema(result, schema("email", "string"), schema("host", "string")); @@ -49,7 +50,10 @@ public void testGrokEmail() throws IOException { public void testGrokAddressOverriding() throws IOException { JSONObject preGrokResult = executeQuery( - String.format(Locale.ROOT, "source = %s | head 3 | fields address", TEST_INDEX_BANK)); + String.format( + Locale.ROOT, + "source = %s | sort account_number | head 3 | fields address", + TEST_INDEX_BANK)); verifySchema(preGrokResult, schema("address", "string")); verifyDataRows( preGrokResult, @@ -61,7 +65,7 @@ public void testGrokAddressOverriding() throws IOException { executeQuery( String.format( Locale.ROOT, - "source = %s | grok address '%s' | head 3 | fields address", + "source = %s | grok address '%s' | sort account_number | head 3 | fields address", TEST_INDEX_BANK, "%{NUMBER} %{GREEDYDATA:address}")); verifySchema(result, schema("address", "string")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java index 46df914e611..fac69220fbf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java @@ -37,7 +37,8 @@ public void testSimplePatternLabelMode_NotShowNumberedToken() throws IOException JSONObject result = executeQuery( String.format( - "source = %s | patterns email mode=label | head 1 | fields email, patterns_field", + "source = %s | patterns email mode=label | sort account_number | head 1 | fields" + + " email, patterns_field", TEST_INDEX_BANK)); verifySchema(result, schema("email", "string"), schema("patterns_field", "string")); verifyDataRows(result, rows("amberduke@pyrami.com", "<*>@<*>.<*>")); @@ -48,8 +49,8 @@ public void testSimplePatternLabelMode_ShowNumberedToken() throws IOException { JSONObject result = executeQuery( String.format( - "source = %s | patterns email mode=label show_numbered_token=true | head 1 | fields" - + " email, patterns_field, tokens", + "source = %s | patterns email mode=label show_numbered_token=true | sort" + + " account_number | head 1 | fields email, patterns_field, tokens", TEST_INDEX_BANK)); verifySchema( result, @@ -100,7 +101,7 @@ public void testSimplePatternLabelModeWithCustomPattern_ShowNumberedToken() thro executeQuery( String.format( "source = %s | patterns email mode=label show_numbered_token=true pattern='@.*' |" - + " head 1 | fields email, patterns_field, tokens", + + " sort account_number | head 1 | fields email, patterns_field, tokens", TEST_INDEX_BANK)); verifySchema( result, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java index 8d31354aa90..47e28c3a827 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLTrendlineIT.java @@ -33,8 +33,8 @@ public void testTrendlineSma() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(3, balance) as balance_trend |" - + " fields balance_trend", + "source=%s | where balance > 30000 | sort account_number | trendline sma(3," + + " balance) as balance_trend | fields balance_trend", TEST_INDEX_BANK)); verifySchema(result, schema("balance_trend", "double")); verifyDataRows( @@ -46,8 +46,8 @@ public void testTrendlineWma() throws IOException { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline wma(3, balance) as balance_trend |" - + " fields balance_trend", + "source=%s | where balance > 30000 | sort account_number | trendline wma(3," + + " balance) as balance_trend | fields balance_trend", TEST_INDEX_BANK)); verifySchema(result, schema("balance_trend", "double")); verifyDataRows( @@ -59,8 +59,8 @@ public void testTrendlineMultipleFields() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(2, balance) as sma wma(3," - + " balance) as wma | fields balance, sma, wma", + "source=%s | where balance > 30000 | sort account_number | trendline sma(2," + + " balance) as sma wma(3, balance) as wma | fields balance, sma, wma", TEST_INDEX_BANK)); verifySchema( result, schema("balance", "bigint"), schema("sma", "double"), schema("wma", "double")); @@ -77,8 +77,8 @@ public void testTrendlineNoAlias() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(2, balance) | fields" - + " balance, balance_trendline", + "source=%s | where balance > 30000 | sort account_number | trendline sma(2," + + " balance) | fields balance, balance_trendline", TEST_INDEX_BANK)); verifySchema(result, schema("balance", "bigint"), schema("balance_trendline", "double")); verifyDataRows( @@ -90,8 +90,8 @@ public void testTrendlineOverwritesExisingField() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | where balance > 30000 | trendline sma(2, balance) as balance | fields" - + " balance", + "source=%s | where balance > 30000 | sort account_number | trendline sma(2," + + " balance) as balance | fields balance", TEST_INDEX_BANK)); verifySchema(result, schema("balance", "double")); verifyDataRows(result, rows((Object) null), rows(36031.5), rows(36689), rows(44313)); @@ -115,7 +115,8 @@ public void testTrendlinePreFilterNullValues() throws Exception { JSONObject result = executeQuery( String.format( - "source=%s | trendline sma(2, balance) | fields" + " balance, balance_trendline", + "source=%s | sort account_number | trendline sma(2, balance) | fields" + + " balance, balance_trendline", TEST_INDEX_BANK_WITH_NULL_VALUES)); verifySchema(result, schema("balance", "bigint"), schema("balance_trendline", "double")); verifyDataRows( diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java index 77daf6dce48..27200078562 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStreamstatsCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.*; @@ -507,6 +508,10 @@ public void testStreamstatsCurrentAndWindowWithNull() throws IOException { @Test public void testStreamstatsGlobal() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 5; Request insertRequest = new Request( @@ -666,6 +671,10 @@ public void testStreamstatsGlobalWithNullBucket() throws IOException { @Test public void testStreamstatsReset() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 5; Request insertRequest = new Request( @@ -934,6 +943,10 @@ public void testMultipleStreamstatsWithNull1() throws IOException { @Test public void testMultipleStreamstatsWithNull2() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 5; Request insertRequest = new Request( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java new file mode 100644 index 00000000000..36d2c4ed82a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -0,0 +1,224 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.legacy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Set; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Test; +import org.opensearch.sql.legacy.TestUtils.AnalyticsIndexConfig; + +/** + * Pure-logic coverage for the analytics-engine field strip (no filesystem / cluster). Verifies that + * nested/geo_point/geo_shape/alias fields are removed from mappings and bulk data on the AE route — + * including nested paths and arrays of objects — that a supported {@code binary} field is + * left intact, and that everything is a no-op when the route is off. + */ +public class AnalyticsFieldStripTests { + + @After + public void clearFlag() { + System.clearProperty(AnalyticsIndexConfig.ENABLED_PROP); + } + + private void enable() { + System.setProperty(AnalyticsIndexConfig.ENABLED_PROP, "true"); + } + + private static List path(String... parts) { + return List.of(parts); + } + + private static final String MAPPING = + "{\"mappings\":{\"properties\":{" + + "\"keep_text\":{\"type\":\"text\"}," + + "\"nested_value\":{\"type\":\"nested\"}," + + "\"geo_point_value\":{\"type\":\"geo_point\"}," + + "\"geo_shape_value\":{\"type\":\"geo_shape\"}," + // default binary (store:false) — parquet store can't create it → stripped. + + "\"binary_value\":{\"type\":\"binary\"}," + // binary with store:true — engine reads it as VARBINARY → kept. + + "\"binary_stored\":{\"type\":\"binary\",\"store\":true}," + + "\"alias_value\":{\"type\":\"alias\",\"path\":\"keep_text\"}," + + "\"obj\":{\"properties\":{" + + " \"keep_inner\":{\"type\":\"keyword\"}," + + " \"inner_geo\":{\"type\":\"geo_point\"}}}" + + "}}}"; + + @Test + public void mappingStrip_removesUnsupportedRecursively_andReportsExactPaths() { + enable(); + JSONObject json = new JSONObject(MAPPING); + Set> dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + + assertEquals( + Set.of( + path("nested_value"), + path("geo_point_value"), + path("geo_shape_value"), + // default binary (store:false) is dropped — parquet store can't create it. + path("binary_value"), + path("alias_value"), + // recursive: the geo_point under the object reports its full path. + path("obj", "inner_geo")), + dropped); + + JSONObject props = json.getJSONObject("mappings").getJSONObject("properties"); + assertTrue("supported scalar kept", props.has("keep_text")); + assertTrue("store:true binary is scannable — must be kept", props.has("binary_stored")); + assertFalse( + "store:false binary not creatable on parquet store — dropped", props.has("binary_value")); + assertFalse(props.has("nested_value")); + assertFalse(props.has("geo_point_value")); + assertFalse(props.has("geo_shape_value")); + assertFalse(props.has("alias_value")); + + // object field kept, but its unsupported sub-property stripped recursively + assertTrue(props.has("obj")); + JSONObject inner = props.getJSONObject("obj").getJSONObject("properties"); + assertTrue(inner.has("keep_inner")); + assertFalse("nested geo_point inside object dropped", inner.has("inner_geo")); + } + + /** + * complex_geo-shaped coverage (the case the top-level-only strip used to miss): a {@code + * geo_point} buried under object fields. Mapping must drop the exact nested path; bulk source + * must drop the same nested value while keeping unaffected siblings. + */ + @Test + public void complexGeoShape_stripsNestedPath_andKeepsSiblings() { + enable(); + String mapping = + "{\"mappings\":{\"properties\":{" + + "\"location\":{\"properties\":{" + + " \"point\":{\"type\":\"geo_point\"}," + + " \"name\":{\"type\":\"text\"}," + + " \"city\":{\"type\":\"keyword\"}," + + " \"country\":{\"type\":\"keyword\"}}}," + + "\"nested_locations\":{\"properties\":{" + + " \"primary\":{\"properties\":{\"office\":{\"type\":\"geo_point\"}}}}}" + + "}}}"; + JSONObject json = new JSONObject(mapping); + Set> dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + + // exact paths, not top-level names + assertEquals( + Set.of(path("location", "point"), path("nested_locations", "primary", "office")), dropped); + + JSONObject loc = json.getJSONObject("mappings").getJSONObject("properties"); + JSONObject locProps = loc.getJSONObject("location").getJSONObject("properties"); + assertFalse("location.point removed", locProps.has("point")); + assertTrue("location.name kept", locProps.has("name")); + assertTrue("location.city kept", locProps.has("city")); + assertTrue("location.country kept", locProps.has("country")); + + // now strip a matching source doc by those exact paths + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"location\":{\"point\":{\"lat\":1,\"lon\":2},\"name\":\"hq\",\"city\":\"sea\"," + + "\"country\":\"us\"}," + + "\"nested_locations\":{\"primary\":{\"office\":{\"lat\":3,\"lon\":4}}}}\n"; + String out = AnalyticsIndexConfig.stripBulkFields(bulk, dropped); + JSONObject doc = new JSONObject(out.split("\n")[1]); + JSONObject docLoc = doc.getJSONObject("location"); + assertFalse("source location.point removed", docLoc.has("point")); + assertTrue("source location.name kept", docLoc.has("name")); + assertTrue("source location.city kept", docLoc.has("city")); + assertTrue("source location.country kept", docLoc.has("country")); + assertFalse( + "source nested_locations.primary.office removed", + doc.getJSONObject("nested_locations").getJSONObject("primary").has("office")); + } + + /** A dropped path may sit under an array of objects — every element must be stripped. */ + @Test + public void bulkStrip_handlesArraysOfObjects() { + enable(); + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"offices\":[{\"loc\":{\"lat\":1},\"name\":\"a\"},{\"loc\":{\"lat\":2}," + + "\"name\":\"b\"}]}\n"; + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("offices", "loc"))); + JSONObject doc = new JSONObject(out.split("\n")[1]); + var arr = doc.getJSONArray("offices"); + for (int i = 0; i < arr.length(); i++) { + JSONObject office = arr.getJSONObject(i); + assertFalse("offices[" + i + "].loc removed", office.has("loc")); + assertTrue("offices[" + i + "].name kept", office.has("name")); + } + } + + @Test + public void mappingStrip_noopWhenDisabled() { + JSONObject json = new JSONObject(MAPPING); + Set> dropped = AnalyticsIndexConfig.stripUnsupportedMappingFields(json); + assertTrue(dropped.isEmpty()); + assertTrue(json.getJSONObject("mappings").getJSONObject("properties").has("nested_value")); + } + + @Test + public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { + enable(); + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"geo_shape_value\":\"POINT(1" + + " 2)\"}\n" + + "{\"index\":{\"_id\":\"2\"}}\n" + + "{\"keep_text\":\"y\",\"nested_value\":[{\"a\":1}]}\n"; + String out = + AnalyticsIndexConfig.stripBulkFields( + bulk, Set.of(path("geo_point_value"), path("geo_shape_value"), path("nested_value"))); + + String[] lines = out.split("\n"); + // action lines untouched + assertTrue(lines[0].contains("\"index\"")); + assertTrue(lines[2].contains("\"index\"")); + // source lines stripped, supported field retained + JSONObject doc1 = new JSONObject(lines[1]); + assertTrue(doc1.has("keep_text")); + assertFalse(doc1.has("geo_point_value")); + assertFalse(doc1.has("geo_shape_value")); + JSONObject doc2 = new JSONObject(lines[3]); + assertTrue(doc2.has("keep_text")); + assertFalse(doc2.has("nested_value")); + } + + @Test + public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { + enable(); + // doc1 carries a dropped key (gets rewritten); doc2 does not (must pass through verbatim). + String docWithDrop = "{\"keep_text\":\"x\",\"nested_value\":[{\"a\":1}]}"; + String docNoDrop = "{\"keep_text\":\"y\",\"age\": 30,\"z\":1}"; + String bulk = + "{\"index\":{\"_id\":\"1\"}}\n" + + docWithDrop + + "\n" + + "{\"index\":{\"_id\":\"2\"}}\n" + + docNoDrop + + "\n"; + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("nested_value"))); + String[] lines = out.split("\n", -1); + // The doc that had no dropped key is byte-for-byte identical (odd spacing/key order preserved). + assertEquals(docNoDrop, lines[3]); + // The doc that had a dropped key lost it. + assertFalse(new JSONObject(lines[1]).has("nested_value")); + } + + @Test + public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { + String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; + // disabled -> unchanged even with a drop set + assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("geo_point_value")))); + // enabled but empty drop set -> unchanged + enable(); + assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of())); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 099b2f7e0cb..67c7e3a9c74 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -66,6 +66,12 @@ public void setUpIndices() throws Exception { initClient(); } + // When -Dtests.analytics.parquet_indices=true, make every index (including ones a test + // auto-creates via a raw document PUT, which bypasses createIndexByRestClient) parquet-backed + // composite, so it is stored as a DataFormatAwareEngine and is actually scannable by the + // analytics engine it routes to. Must run before init() creates any index. + TestUtils.AnalyticsIndexConfig.applyClusterSettings(client()); + if (shouldResetQuerySizeLimit()) { resetQuerySizeLimit(); } @@ -208,7 +214,9 @@ protected synchronized void loadIndex(Index index, RestClient client) throws IOE if (!isIndexExist(client, indexName)) { createIndexByRestClient(client, indexName, mapping); - loadDataByRestClient(client, indexName, dataSet); + // On the analytics-engine route, unsupported-typed fields are stripped from the mapping; drop + // the same keys from the bulk data so the two agree. Empty (no-op) off the AE route. + loadDataByRestClient(client, indexName, dataSet, analyticsDroppedFields(mapping)); } } @@ -533,6 +541,11 @@ public enum Index { "account", getAccountIndexMapping(), "src/test/resources/accounts.json"), + ACCOUNT_EXTENDED( + TestsConstants.TEST_INDEX_ACCOUNT_EXTENDED, + "account_extended", + getAccountExtendedIndexMapping(), + "src/test/resources/accounts_extended.json"), PHRASE( TestsConstants.TEST_INDEX_PHRASE, "phrase", @@ -626,6 +639,11 @@ public enum Index { "account", getBankIndexMapping(), "src/test/resources/bank.json"), + BANK_EXTENDED( + TestsConstants.TEST_INDEX_BANK_EXTENDED, + "bank_extended", + getBankExtendedIndexMapping(), + "src/test/resources/bank_extended.json"), BANK_TWO( TestsConstants.TEST_INDEX_BANK_TWO, "account_two", @@ -716,6 +734,11 @@ public enum Index { "_doc", getDataTypeNonnumericIndexMapping(), "src/test/resources/datatypes.json"), + DATETIME_SIMPLE( + TestsConstants.TEST_INDEX_DATETIME_SIMPLE, + "_doc", + getDateTimeSimpleIndexMapping(), + "src/test/resources/datetime_simple.json"), BEER( TestsConstants.TEST_INDEX_BEER, "beer", null, "src/test/resources/beer.stackexchange.json"), NULL_MISSING( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 681c28c0a05..c478165bf07 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -19,10 +19,13 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.stream.Collectors; +import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -64,10 +67,25 @@ public static final class AnalyticsIndexConfig { */ public static final String ENABLED_PROP = "tests.analytics.parquet_indices"; + /** + * System property overriding the number of primary shards for analytics-backed test indices. + * Defaults to 1 (single-shard). Set to e.g. "3" for multi-shard coverage runs. + */ + public static final String NUM_SHARDS_PROP = "tests.analytics.num_shards"; + public static boolean isEnabled() { return Boolean.parseBoolean(System.getProperty(ENABLED_PROP, "false")); } + public static int getNumShards() { + return Integer.parseInt(System.getProperty(NUM_SHARDS_PROP, "1")); + } + + // Composite-store format values shared by the index-level and cluster-level settings below. + private static final String DATAFORMAT_COMPOSITE = "composite"; + private static final String PRIMARY_FORMAT_PARQUET = "parquet"; + private static final String SECONDARY_FORMAT_LUCENE = "lucene"; + /** * Inject the parquet-backed composite-store index settings into {@code jsonObject}. No-op when * the config is disabled; idempotent — safe on any index-creation JSON shape. @@ -80,15 +98,40 @@ static void applyIndexCreationSettings(JSONObject jsonObject) { jsonObject.has("settings") ? jsonObject.getJSONObject("settings") : new JSONObject(); JSONObject indexSettings = settings.has("index") ? settings.getJSONObject("index") : new JSONObject(); - indexSettings.put("number_of_shards", 1); + indexSettings.put("number_of_shards", getNumShards()); indexSettings.put("pluggable.dataformat.enabled", true); - indexSettings.put("pluggable.dataformat", "composite"); - indexSettings.put("composite.primary_data_format", "parquet"); - indexSettings.put("composite.secondary_data_formats", new org.json.JSONArray().put("lucene")); + indexSettings.put("pluggable.dataformat", DATAFORMAT_COMPOSITE); + indexSettings.put("composite.primary_data_format", PRIMARY_FORMAT_PARQUET); + indexSettings.put( + "composite.secondary_data_formats", new JSONArray().put(SECONDARY_FORMAT_LUCENE)); settings.put("index", indexSettings); jsonObject.put("settings", settings); } + /** + * Set the composite-store defaults at the cluster level so even indices auto-created by a raw + * document {@code PUT} (which bypass {@link #applyIndexCreationSettings}) are parquet-backed. + * Otherwise such an index inherits only the composite value — so it routes to the analytics + * engine — but not the {@code .enabled} flag, leaving it stored as a plain-Lucene {@code + * EngineBackedIndexer} that fails at query time. No-op when disabled; idempotent. + */ + public static void applyClusterSettings(RestClient client) { + if (!isEnabled()) { + return; + } + JSONObject persistent = + new JSONObject() + .put("cluster.pluggable.dataformat.enabled", true) + .put("cluster.pluggable.dataformat", DATAFORMAT_COMPOSITE) + .put("cluster.composite.primary_data_format", PRIMARY_FORMAT_PARQUET) + .put( + "cluster.composite.secondary_data_formats", + new JSONArray().put(SECONDARY_FORMAT_LUCENE)); + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity(new JSONObject().put("persistent", persistent).toString()); + performRequest(client, request); + } + /** * Returns the {@code _bulk} refresh query string for the current index type. Parquet-backed * indices in the analytics-backend-lucene composite engine don't yet implement {@code @@ -102,6 +145,185 @@ static String bulkLoadRefreshParam() { return isEnabled() ? "refresh=true" : "refresh=wait_for&wait_for_active_shards=all"; } + /** + * Field types the analytics-engine (DataFusion) backend cannot read, and which therefore must + * be removed from a test index's mapping (and the matching key from each bulk doc) before the + * index is created/loaded on the analytics-engine route. Seeding a dataset that happens to use + * one of them would otherwise fail ingestion and surface as an unrelated {@code expected:<1> + * but was:<0>} downstream — so we strip uniformly at load time across every dataset, no + * per-index variant files to author or keep in sync. + * + *

    This set is the single source of truth. The strip triggers purely on a mapping's + * field types (not on which IT is running), so any IT — existing or newly added — that + * loads one of these datasets through {@link SQLIntegTestCase#loadIndex} is handled out of the + * box. {@code AnalyticsUnsupportedFieldStripVerifyIT} imports this same constant and proves no + * listed type survives in any live mapping, so the list cannot silently drift. + * + *

    This set is the always-strip core. {@code binary} is handled conditionally and is + * NOT in this set — see {@link #isUnsupportedForAeStore}. + * + *

    What is and isn't here, and why: + * + *

      + *
    • {@code nested} — entire subtree dropped (the engine has no nested-document support). + *
    • {@code geo_point}, {@code geo_shape} — fall through the engine's type whitelist ({@code + * OpenSearchSchemaBuilder.mapFieldType} default → null), so the column can't scan. + *
    • {@code alias} — indirection the scan-row-type builder doesn't resolve. + *
    • {@code binary} is conditional, not in this set. The engine's read path maps + * {@code binary → VARBINARY} (same as {@code ip}), so a binary column scans fine — but + * the parquet/composite store rejects creating a {@code binary} field that lacks + * {@code store: true} ("Unable to derive source for [X] with store disabled", verified on + * the AE cluster). So we strip a {@code binary} field only when it is NOT {@code store: + * true}; a {@code store: true} binary field is kept and ingests/scans normally. Adding + * {@code binary} to this unconditional set would over-strip the supported case. + *
    + */ + public static final Set UNSUPPORTED_FIELD_TYPES = + Set.of("nested", "geo_point", "geo_shape", "alias"); + + /** + * Whether a field definition cannot be created/scanned on the analytics-engine route and so + * must be stripped. True for any {@link #UNSUPPORTED_FIELD_TYPES} type, and additionally for a + * {@code binary} field that is not {@code store: true} — the parquet store can't derive {@code + * _source} for a store-disabled binary field at mapping time. + */ + private static boolean isUnsupportedForAeStore(JSONObject def) { + String type = def.optString("type", null); + if (type == null) { + return false; + } + if (UNSUPPORTED_FIELD_TYPES.contains(type)) { + return true; + } + return "binary".equals(type) && !def.optBoolean("store", false); + } + + /** + * Remove every {@link #UNSUPPORTED_FIELD_TYPES} property (recursively, including object + * sub-properties) from an index-creation JSON's {@code mappings.properties}, and report the + * exact dotted paths that were dropped so the matching values can be stripped from + * bulk data. No-op when disabled. Mutates {@code jsonObject} in place. + * + *

    Paths are returned as ordered part lists, e.g. {@code ["location", "point"]} for a {@code + * geo_point} sub-field of an object {@code location}, so the bulk-source strip can remove the + * exact nested value while preserving unaffected siblings ({@code location.city} etc.). A + * top-level unsupported field comes back as a single-element list, e.g. {@code + * ["nested_value"]}. + * + * @return the dropped field paths (empty when disabled or nothing matched) + */ + static Set> stripUnsupportedMappingFields(JSONObject jsonObject) { + if (!isEnabled() || !jsonObject.has("mappings")) { + return Set.of(); + } + JSONObject mappings = jsonObject.getJSONObject("mappings"); + if (!mappings.has("properties")) { + return Set.of(); + } + Set> dropped = new HashSet<>(); + collectAndRemoveUnsupported(mappings.getJSONObject("properties"), new ArrayList<>(), dropped); + return dropped; + } + + /** + * Walk a {@code properties} block, removing every property whose {@code type} is unsupported + * (recording its full path) and recursing into object sub-{@code properties} for the rest. + */ + private static void collectAndRemoveUnsupported( + JSONObject properties, List prefix, Set> dropped) { + for (String field : properties.keySet().toArray(new String[0])) { + JSONObject def = properties.optJSONObject(field); + if (def == null) { + continue; + } + List path = new ArrayList<>(prefix); + path.add(field); + if (isUnsupportedForAeStore(def)) { + properties.remove(field); + dropped.add(List.copyOf(path)); + } else if (def.has("properties")) { + collectAndRemoveUnsupported(def.getJSONObject("properties"), path, dropped); + } + } + } + + /** + * Strip the given dropped paths from every source document of a bulk NDJSON payload. + * Bulk format alternates an action line ({@code {"index":{...}}}) with a source line; only + * source lines (those without a bulk action key) are rewritten. No-op when disabled or {@code + * droppedPaths} is empty. + * + *

    Each path is removed recursively: it descends through nested objects and arrays of + * objects (so a {@code nested}/object array has the field stripped from every element), + * leaving unaffected siblings intact. A source line is re-serialized only when removing a + * path actually changed it; every other line (action lines and docs that never had the + * dropped path) is appended byte-for-byte unchanged, so untouched docs match the fixture + * exactly. + */ + static String stripBulkFields(String bulkBody, Set> droppedPaths) { + if (!isEnabled() || droppedPaths.isEmpty()) { + return bulkBody; + } + String[] lines = bulkBody.split("\n", -1); + StringBuilder out = new StringBuilder(bulkBody.length()); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + String trimmed = line.trim(); + if (!trimmed.isEmpty() && trimmed.charAt(0) == '{') { + JSONObject doc = new JSONObject(trimmed); + boolean isActionLine = + doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); + if (!isActionLine) { + boolean removedAny = false; + for (List path : droppedPaths) { + removedAny |= removePath(doc, path, 0); + } + // Only rewrite the line if we actually removed something; otherwise leave it verbatim + // so untouched docs stay byte-for-byte identical to the fixture. + if (removedAny) { + line = doc.toString(); + } + } + } + out.append(line); + if (i < lines.length - 1) { + out.append('\n'); + } + } + return out.toString(); + } + + /** + * Remove {@code path[idx..]} from {@code node}, descending through objects and arrays of + * objects. Returns true if anything was removed. At the last path part the key is deleted from + * the object; otherwise we recurse into the child object (or each object element of a child + * array). + */ + private static boolean removePath(Object node, List path, int idx) { + String part = path.get(idx); + boolean last = idx == path.size() - 1; + boolean removed = false; + if (node instanceof JSONObject) { + JSONObject obj = (JSONObject) node; + if (!obj.has(part)) { + return false; + } + if (last) { + obj.remove(part); + return true; + } + removed = removePath(obj.get(part), path, idx + 1); + } else if (node instanceof JSONArray) { + // The mapping path can sit under an array (e.g. a nested/object array); strip from each + // object element, ignoring non-object elements. + JSONArray arr = (JSONArray) node; + for (int j = 0; j < arr.length(); j++) { + removed |= removePath(arr.get(j), path, idx); + } + } + return removed; + } + private AnalyticsIndexConfig() {} } @@ -117,10 +339,27 @@ public static void createIndexByRestClient(RestClient client, String indexName, JSONObject jsonObject = isNullOrEmpty(mapping) ? new JSONObject() : new JSONObject(mapping); setZeroReplicas(jsonObject); AnalyticsIndexConfig.applyIndexCreationSettings(jsonObject); + // On the analytics-engine route, drop fields of types DataFusion can't read so the index can + // still be created from datasets that happen to use them. No-op otherwise. + AnalyticsIndexConfig.stripUnsupportedMappingFields(jsonObject); request.setJsonEntity(jsonObject.toString()); performRequest(client, request); } + /** + * Exact dropped field paths the analytics-engine route would remove from {@code mapping} + * (e.g. {@code ["location", "point"]} for a nested {@code geo_point}). Used by {@link + * #loadDataByRestClient(RestClient, String, String, java.util.Set)} so bulk docs drop the same + * paths the mapping dropped. Returns an empty set when AE is disabled or {@code mapping} is + * empty. + */ + public static Set> analyticsDroppedFields(String mapping) { + if (isNullOrEmpty(mapping)) { + return Set.of(); + } + return AnalyticsIndexConfig.stripUnsupportedMappingFields(new JSONObject(mapping)); + } + /** * Sets number_of_replicas to 0 in the index settings. This makes multi-node behavior consistent * (4261) and prevents tests @@ -184,12 +423,60 @@ public static boolean isIndexExist(RestClient client, String indexName) { */ public static void loadDataByRestClient( RestClient client, String indexName, String dataSetFilePath) throws IOException { + loadDataByRestClient(client, indexName, dataSetFilePath, Set.of()); + } + + /** + * Same as {@link #loadDataByRestClient(RestClient, String, String)} but strips {@code + * droppedPaths} (the exact field paths removed from the mapping on the analytics-engine route) + * from every bulk source doc, so the index mapping and the data agree. When AE is disabled or + * {@code droppedPaths} is empty this is byte-for-byte identical to the 3-arg form. + */ + public static void loadDataByRestClient( + RestClient client, String indexName, String dataSetFilePath, Set> droppedPaths) + throws IOException { Path path = Paths.get(getResourceFilePath(dataSetFilePath)); + String body = new String(Files.readAllBytes(path)); + body = AnalyticsIndexConfig.stripBulkFields(body, droppedPaths); Request request = new Request( "POST", "/" + indexName + "/_bulk?" + AnalyticsIndexConfig.bulkLoadRefreshParam()); - request.setJsonEntity(new String(Files.readAllBytes(path))); - performRequest(client, request); + request.setJsonEntity(body); + Response response = performRequest(client, request); + failIfBulkHadItemErrors(indexName, response); + } + + /** + * A {@code _bulk} call can return HTTP 200 while individual items failed ({@code "errors":true}). + * That silent partial ingestion is exactly the failure mode that surfaces later as an unrelated + * row-count/assertion error in some downstream IT, so fail loudly here — naming the index and the + * first few item errors — for ALL test bulk loads, not just the analytics-engine route. + */ + private static void failIfBulkHadItemErrors(String indexName, Response response) + throws IOException { + JSONObject body = new JSONObject(getResponseBody(response)); + if (!body.optBoolean("errors", false)) { + return; + } + JSONArray items = body.optJSONArray("items"); + List firstErrors = new ArrayList<>(); + if (items != null) { + for (int i = 0; i < items.length() && firstErrors.size() < 5; i++) { + JSONObject action = items.getJSONObject(i); + // each item is { "": { ..., "error": {...} } } where is index/create/update/delete + for (String op : action.keySet()) { + JSONObject result = action.optJSONObject(op); + if (result != null && result.has("error")) { + firstErrors.add("doc#" + i + ": " + result.get("error")); + } + } + } + } + throw new IllegalStateException( + "Bulk load into [" + + indexName + + "] had item failures (errors=true). First failures:\n " + + String.join("\n ", firstErrors)); } /** @@ -231,6 +518,11 @@ public static String getAccountIndexMapping() { return getMappingFile(mappingFile); } + public static String getAccountExtendedIndexMapping() { + String mappingFile = "account_extended_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getPhraseIndexMapping() { String mappingFile = "phrase_index_mapping.json"; return getMappingFile(mappingFile); @@ -303,6 +595,11 @@ public static String getBankIndexMapping() { return getMappingFile(mappingFile); } + public static String getBankExtendedIndexMapping() { + String mappingFile = "bank_extended_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getGeoIpIndexMapping() { String mappingFile = "geoip_index_mapping.json"; return getMappingFile(mappingFile); @@ -363,6 +660,11 @@ public static String getDataTypeNonnumericIndexMapping() { return getMappingFile(mappingFile); } + public static String getDateTimeSimpleIndexMapping() { + String mappingFile = "datetime_simple_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getGeopointIndexMapping() { String mappingFile = "geopoint_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 574f7b1eedb..5d7eeb328af 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -15,6 +15,7 @@ public class TestsConstants { public static final String TEST_INDEX_ONLINE = TEST_INDEX + "_online"; public static final String TEST_INDEX_ACCOUNT = TEST_INDEX + "_account"; + public static final String TEST_INDEX_ACCOUNT_EXTENDED = TEST_INDEX_ACCOUNT + "_extended"; public static final String TEST_INDEX_PHRASE = TEST_INDEX + "_phrase"; public static final String TEST_INDEX_DOG = TEST_INDEX + "_dog"; public static final String TEST_INDEX_DOG2 = TEST_INDEX + "_dog2"; @@ -39,6 +40,7 @@ public class TestsConstants { public static final String TEST_INDEX_JOIN_TYPE = TEST_INDEX + "_join_type"; public static final String TEST_INDEX_UNEXPANDED_OBJECT = TEST_INDEX + "_unexpanded_object"; public static final String TEST_INDEX_BANK = TEST_INDEX + "_bank"; + public static final String TEST_INDEX_BANK_EXTENDED = TEST_INDEX_BANK + "_extended"; public static final String TEST_INDEX_BANK_TWO = TEST_INDEX_BANK + "_two"; public static final String TEST_INDEX_BANK_WITH_NULL_VALUES = TEST_INDEX_BANK + "_with_null_values"; @@ -59,6 +61,7 @@ public class TestsConstants { public static final String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; public static final String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; public static final String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; + public static final String TEST_INDEX_DATETIME_SIMPLE = TEST_INDEX + "_datetime_simple"; public static final String TEST_INDEX_BEER = TEST_INDEX + "_beer"; public static final String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; public static final String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index 5cd89fa7cd9..f6ec903c395 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,17 +5,55 @@ package org.opensearch.sql.plugin; +import static org.junit.Assume.assumeTrue; + +import java.io.IOException; +import org.junit.Before; import org.junit.Test; -import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.sql.legacy.OpenSearchSQLRestTestCase; +import org.opensearch.sql.legacy.TestUtils; /** * Smoke test: verifies that opensearch-sql loads cleanly alongside arrow-flight-rpc and * analytics-engine. A successful cluster start is the only assertion — no sql-specific logic runs. + * + *

    This test is only meaningful when the analytics-engine plugin is installed; the dedicated + * {@code :integ-test:analyticsEngineCompatIT} Gradle task bundles the plugin stack for exactly that + * purpose. Other lanes can still discover this class — notably the distribution integ-test + * pipeline, which scans every {@code *IT} class against the built distribution and may run with the + * security plugin enabled and without analytics-engine. A {@code build.gradle} exclude does not + * protect against that pipeline, so the test guards itself instead: + * + *

      + *
    • It extends {@link OpenSearchSQLRestTestCase} so the REST client honours the {@code https}, + * {@code user}, and {@code password} system properties of a secured cluster (a bare {@code + * OpenSearchRestTestCase} speaks plain HTTP and gets its connection closed during client init + * on a TLS-secured port). + *
    • It skips via an assumption when analytics-engine is absent, so a build without the plugin + * reports the test as skipped rather than failed. + *
    */ -public class AnalyticsEngineCompatIT extends OpenSearchRestTestCase { +public class AnalyticsEngineCompatIT extends OpenSearchSQLRestTestCase { + + /** + * Skips the suite unless the analytics-engine plugin is installed. Runs after the framework has + * established the (security-aware) REST client, so the lookup itself succeeds on both plain and + * secured clusters. + */ + @Before + public void requireAnalyticsEngine() throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/plugins?h=component")); + String installedPlugins = TestUtils.getResponseBody(response, true); + assumeTrue( + "analytics-engine plugin not installed — skipping coexistence smoke test", + installedPlugins.contains("analytics-engine")); + } @Test public void testClusterStarted() { - // If the cluster booted, all three plugins loaded without classloader errors. + // If the cluster booted with analytics-engine present, all plugins loaded without classloader + // errors. The assumption above guarantees we only assert this where it is meaningful. } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java index 1cef2a43001..e31a73d91a6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeComparisonIT.java @@ -7,7 +7,7 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.$; import static com.carrotsearch.randomizedtesting.RandomizedTest.$$; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATETIME_SIMPLE; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -28,7 +28,7 @@ public class DateTimeComparisonIT extends PPLIntegTestCase { @Override public void init() throws Exception { super.init(); - loadIndex(Index.DATA_TYPE_NONNUMERIC); + loadIndex(Index.DATETIME_SIMPLE); } private final TimeZone testTz = TimeZone.getDefault(); @@ -231,7 +231,7 @@ public static Iterable compareLtTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') < DATE('1961-04-12')", "ts_d_f", false), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2020-09-16 10:20:30') < TIME('09:07:00')", "ts_t_t", true), - $("TIME('09:07:00') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('09:07:00') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') < TIME('10:20:30')", "ts_t_f", false), $("TIME('20:50:40') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -240,13 +240,13 @@ public static Iterable compareLtTimestampWithOtherTypes() { public static Iterable compareLtDateWithOtherTypes() { return Arrays.asList( $$( - $("DATE('2020-09-16') < TIMESTAMP('3077-04-12 09:07:00')", "d_ts_t", true), + $("DATE('2020-09-16') < TIMESTAMP('2242-04-12 09:07:00')", "d_ts_t", true), $("TIMESTAMP('1961-04-12 09:07:00') < DATE('1984-12-15')", "ts_d_t", true), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') < DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') < TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') < TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') < DATE('2020-09-16')", "t_d_f", false))); } @@ -255,14 +255,14 @@ public static Iterable compareLtTimeWithOtherTypes() { var today = LocalDate.now().toString(); return Arrays.asList( $$( - $("TIME('22:15:07') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('22:15:07') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('1984-12-15 10:20:30') < TIME('10:20:30')", "ts_t_t", true), $("TIME('10:20:30') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') < TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') < DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') < TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') < TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -274,10 +274,10 @@ public static Iterable compareGtTimestampWithOtherTypes() { $("DATE('2020-09-16') > TIMESTAMP('2020-09-15 22:15:07')", "d_ts_t", true), $("TIMESTAMP('2020-09-16 10:20:30') > DATE('2077-04-12')", "ts_d_f", false), $("DATE('1961-04-12') > TIMESTAMP('1961-04-12 00:00:00')", "d_ts_f", false), - $("TIMESTAMP('3077-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), + $("TIMESTAMP('2242-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') > TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') > TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') > TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -288,10 +288,10 @@ public static Iterable compareGtDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') > DATE('2020-09-16')", "ts_d_t", true), $("DATE('2020-09-16') > TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') > DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') > TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') > TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') > DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -304,8 +304,8 @@ public static Iterable compareGtTimeWithOtherTypes() { $("TIME('10:20:30') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') > TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') > DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') > TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') > TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false))); } @@ -319,8 +319,8 @@ public static Iterable compareLteTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') <= DATE('1961-04-12')", "ts_d_f", false), $("DATE('2077-04-12') <= TIMESTAMP('1984-12-15 22:15:07')", "d_ts_f", false), $("TIMESTAMP('" + today + " 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), - $("TIME('09:07:00') <= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), - $("TIMESTAMP('3077-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), + $("TIME('09:07:00') <= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), + $("TIMESTAMP('2242-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), $("TIME('20:50:40') <= TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -333,8 +333,8 @@ public static Iterable compareLteDateWithOtherTypes() { $("DATE('2020-09-16') <= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') <= DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') <= TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') <= TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') <= DATE('2020-09-16')", "t_d_f", false))); } @@ -347,10 +347,10 @@ public static Iterable compareLteTimeWithOtherTypes() { $("TIMESTAMP('1984-12-15 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), $("TIME('22:15:07') <= TIMESTAMP('1984-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') <= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') <= DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') <= TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') <= TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -365,7 +365,7 @@ public static Iterable compareGteTimestampWithOtherTypes() { $("TIMESTAMP('" + today + " 10:20:30') >= TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('1977-07-08 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -376,10 +376,10 @@ public static Iterable compareGteDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') >= DATE('2020-09-16')", "ts_d_t", true), $("DATE('1961-04-12') >= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') >= DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') >= TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') >= TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') >= DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -389,11 +389,11 @@ public static Iterable compareGteTimeWithOtherTypes() { $$( $("TIME('10:20:30') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 20:50:42') >= TIME('10:20:30')", "ts_t_t", true), - $("TIME('22:15:07') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false), + $("TIME('22:15:07') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') >= DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') >= TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') >= TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false))); } @@ -403,7 +403,7 @@ public void testCompare() throws IOException { executeQuery( String.format( "source=%s | eval `%s` = %s | fields `%s`", - TEST_INDEX_DATATYPE_NONNUMERIC, name, functionCall, name)); + TEST_INDEX_DATETIME_SIMPLE, name, functionCall, name)); verifySchema(result, schema(name, null, "boolean")); verifyDataRows(result, rows(expectedResult)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index cbfe3c84464..aa5b7758446 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -1565,6 +1566,10 @@ public void testExtract() throws IOException { @Test public void testCompareAgainstUTCDate() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); String isoTimestamp = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'")); String pplTimestamp = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index 3f078fe6512..4d755c1ab77 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -16,9 +16,12 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.stream.Stream; import org.json.JSONObject; import org.junit.Ignore; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; public class FieldsCommandIT extends PPLIntegTestCase { @@ -26,25 +29,37 @@ public class FieldsCommandIT extends PPLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.ACCOUNT); + loadIndex(Index.ACCOUNT_EXTENDED); loadIndex(Index.BANK); + loadIndex(Index.BANK_EXTENDED); loadIndex(Index.MERGE_TEST_1); loadIndex(Index.MERGE_TEST_2); } - @Test - public void testBasicFieldSelection() throws IOException { - JSONObject result = - executeQuery(String.format("source=%s | fields firstname, lastname", TEST_INDEX_ACCOUNT)); + // --- Parameterized sources --- + + static Stream accountIndexSources() { + return sourceViews(TEST_INDEX_ACCOUNT); + } + + static Stream bankIndexSources() { + return sourceViews(TEST_INDEX_BANK); + } + + // --- Tests --- + + @ParameterizedTest(name = "querySource={0}") + @MethodSource("accountIndexSources") + public void testBasicFieldSelection(String querySource) throws IOException { + JSONObject result = executeQuery(querySource + " | fields firstname, lastname"); verifyColumn(result, columnName("firstname"), columnName("lastname")); verifySchema(result, schema("firstname", "string"), schema("lastname", "string")); } - @Test - public void testMultipleFieldSelection() throws IOException { - JSONObject result = - executeQuery( - String.format( - "source=%s | fields firstname, lastname, age | head 3", TEST_INDEX_ACCOUNT)); + @ParameterizedTest(name = "querySource={0}") + @MethodSource("accountIndexSources") + public void testMultipleFieldSelection(String querySource) throws IOException { + JSONObject result = executeQuery(querySource + " | fields firstname, lastname, age | head 3"); verifySchema( result, schema("firstname", "string"), @@ -57,12 +72,11 @@ public void testMultipleFieldSelection() throws IOException { rows("Nanette", "Bates", 28)); } - @Test - public void testSpecialDataTypes() throws IOException { - JSONObject result = - executeQuery(String.format("source=%s | fields birthdate", TEST_INDEX_BANK)); + @ParameterizedTest(name = "querySource={0}") + @MethodSource("bankIndexSources") + public void testSpecialDataTypes(String querySource) throws IOException { + JSONObject result = executeQuery(querySource + " | fields birthdate"); verifySchema(result, schema("birthdate", null, "timestamp")); - verifyDataRows( result, rows("2017-10-23 00:00:00"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java index a24896adaf3..06fb7b627f9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/HeadCommandIT.java @@ -38,27 +38,32 @@ public void init() throws Exception { @Test public void testHead() throws IOException { JSONObject result = - executeQuery(String.format("source=%s | fields firstname, age | head", TEST_INDEX_ACCOUNT)); + executeQuery( + String.format( + "source=%s | sort account_number | fields firstname, age | head", + TEST_INDEX_ACCOUNT)); verifyDataRows( result, + rows("Bradshaw", 29), rows("Amber", 32), + rows("Roberta", 22), + rows("Levine", 26), + rows("Rodriquez", 31), + rows("Leola", 30), rows("Hattie", 36), - rows("Nanette", 28), - rows("Dale", 33), - rows("Elinor", 36), - rows("Virginia", 39), - rows("Dillard", 34), - rows("Mcgee", 39), - rows("Aurelia", 37), - rows("Fulton", 23)); + rows("Levy", 22), + rows("Jan", 35), + rows("Opal", 39)); } @Test public void testHeadWithNumber() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | fields firstname, age | head 3", TEST_INDEX_ACCOUNT)); - verifyDataRows(result, rows("Amber", 32), rows("Hattie", 36), rows("Nanette", 28)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Bradshaw", 29), rows("Amber", 32), rows("Roberta", 22)); } @Ignore("Fix https://github.com/opensearch-project/sql/issues/703#issuecomment-1211422130") @@ -87,24 +92,26 @@ public void testHeadWithNumberLargerThanMaxResultWindow() throws IOException { setMaxResultWindow(TEST_INDEX_ACCOUNT, 10); JSONObject result = executeQuery( - String.format("source=%s | fields firstname, age | head 15", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 15", + TEST_INDEX_ACCOUNT)); verifyDataRows( result, + rows("Bradshaw", 29), rows("Amber", 32), + rows("Roberta", 22), + rows("Levine", 26), + rows("Rodriquez", 31), + rows("Leola", 30), rows("Hattie", 36), + rows("Levy", 22), + rows("Jan", 35), + rows("Opal", 39), + rows("Dominique", 37), + rows("Jenkins", 20), + rows("Stafford", 20), rows("Nanette", 28), - rows("Dale", 33), - rows("Elinor", 36), - rows("Virginia", 39), - rows("Dillard", 34), - rows("Mcgee", 39), - rows("Aurelia", 37), - rows("Fulton", 23), - rows("Burton", 31), - rows("Josie", 32), - rows("Hughes", 30), - rows("Hall", 25), - rows("Deidre", 33)); + rows("Erma", 39)); } @Ignore("Fix https://github.com/opensearch-project/sql/issues/703#issuecomment-1211422130") @@ -138,7 +145,9 @@ public void testHeadWithLargeNumber() throws IOException { public void testHeadWithNumberAndFrom() throws IOException { JSONObject result = executeQuery( - String.format("source=%s | fields firstname, age | head 3 from 4", TEST_INDEX_ACCOUNT)); - verifyDataRows(result, rows("Elinor", 36), rows("Virginia", 39), rows("Dillard", 34)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3 from 4", + TEST_INDEX_ACCOUNT)); + verifyDataRows(result, rows("Rodriquez", 31), rows("Leola", 30), rows("Hattie", 36)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 6c25c415af1..3db2142cffe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -6,6 +6,10 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT_EXTENDED; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_EXTENDED; import static org.opensearch.sql.plugin.rest.RestPPLQueryAction.QUERY_API_ENDPOINT; import com.google.common.io.Resources; @@ -14,6 +18,8 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Locale; +import java.util.Map; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONException; @@ -148,6 +154,35 @@ protected static String source(String index, String query) { return String.format("source=%s | %s", index, query); } + // Maps each base index to its extended counterpart with synthetic rows. + private static final Map INDEX_VIEWS = + Map.of( + TEST_INDEX_ACCOUNT, TEST_INDEX_ACCOUNT_EXTENDED, + TEST_INDEX_BANK, TEST_INDEX_BANK_EXTENDED); + + /** + * Returns a WHERE-prefix view source string over an extended index. The extended index is + * expected to contain all rows from the base index with {@code "_is_real":true}, plus additional + * synthetic rows with {@code "_is_real":false}. Use this as a drop-in replacement for {@code + * "source="} in parameterized tests to verify commands work correctly when preceded by a + * {@code where} filter. + */ + protected static String sourceView(String extendedIndex) { + return String.format("source=%s | where _is_real | fields - _is_real", extendedIndex); + } + + /** + * Returns a stream of parameterized source strings for the given base index: a direct source and, + * if an extended view is registered in {@link #INDEX_VIEWS}, a WHERE-prefix view over it. New + * views can be added by updating {@link #INDEX_VIEWS} without changing individual test files. + */ + protected static Stream sourceViews(String baseIndex) { + String extendedIndex = INDEX_VIEWS.get(baseIndex); + return extendedIndex != null + ? Stream.of(String.format("source=%s", baseIndex), sourceView(extendedIndex)) + : Stream.of(String.format("source=%s", baseIndex)); + } + protected void timing(MapBuilder builder, String query, String ppl) throws IOException { executeQuery(ppl); // warm-up diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java index b8e5b2a5722..c5360a5dfa0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl; +import static org.junit.Assume.assumeFalse; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.columnName; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -1053,6 +1054,10 @@ public void testSearchWithNumericTimeRange() throws IOException { @Test public void testSearchTimeModifierWithSnappedWeek() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); // Test whether alignment to weekday works final int docId = 101; @@ -1141,6 +1146,10 @@ public void testSearchTimeModifierWithSnappedWeek() throws IOException { @Test public void testSearchWithRelativeTimeModifiers() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 101; LocalDateTime currentTime = LocalDateTime.now(ZoneOffset.UTC); @@ -1189,6 +1198,10 @@ public void testSearchWithRelativeTimeModifiers() throws IOException { @Test public void testSearchWithTimeUnitSnapping() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 101; LocalDateTime currentHour = LocalDateTime.now(ZoneOffset.UTC).truncatedTo(ChronoUnit.HOURS); @@ -1237,6 +1250,10 @@ public void testSearchWithTimeUnitSnapping() throws IOException { @Test public void testSearchWithQuarterlyModifiers() throws IOException { + assumeFalse( + "Test mutates docs via PUT+DELETE, which DataFormatAwareEngine" + + " (analytics-engine storage path) does not support.", + isAnalyticsParquetIndicesEnabled()); final int docId = 101; LocalDateTime currentQuarter = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java index 8b5df6e710e..3d16bd034bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TrailingPipeIT.java @@ -34,8 +34,10 @@ public void init() throws Exception { @Test public void testTrailingPipeAfterSource() throws IOException { // Query with trailing pipe should produce same results as without - JSONObject resultWithout = executeQuery(String.format("source=%s", TEST_INDEX_ACCOUNT)); - JSONObject resultWith = executeQuery(String.format("source=%s |", TEST_INDEX_ACCOUNT)); + JSONObject resultWithout = + executeQuery(String.format("source=%s | sort account_number", TEST_INDEX_ACCOUNT)); + JSONObject resultWith = + executeQuery(String.format("source=%s | sort account_number |", TEST_INDEX_ACCOUNT)); // Both should return the same data assertTrue(resultWithout.similar(resultWith)); @@ -52,11 +54,13 @@ public void testTrailingPipeAfterFields() throws IOException { JSONObject resultWithout = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age", TEST_INDEX_ACCOUNT)); + "source=%s | where age > 30 | sort account_number | fields firstname, age", + TEST_INDEX_ACCOUNT)); JSONObject resultWith = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age |", TEST_INDEX_ACCOUNT)); + "source=%s | where age > 30 | sort account_number | fields firstname, age |", + TEST_INDEX_ACCOUNT)); assertTrue(resultWithout.similar(resultWith)); } @@ -71,10 +75,14 @@ public void testTrailingPipeAfterFields() throws IOException { public void testTrailingPipeAfterHead() throws IOException { JSONObject resultWithout = executeQuery( - String.format("source=%s | fields firstname, age | head 3", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3", + TEST_INDEX_ACCOUNT)); JSONObject resultWith = executeQuery( - String.format("source=%s | fields firstname, age | head 3 |", TEST_INDEX_ACCOUNT)); + String.format( + "source=%s | sort account_number | fields firstname, age | head 3 |", + TEST_INDEX_ACCOUNT)); assertTrue(resultWithout.similar(resultWith)); } @@ -115,11 +123,13 @@ public void testEmptyPipeInMiddle() throws IOException { JSONObject resultNormal = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age", TEST_INDEX_ACCOUNT)); + "source=%s | where age > 30 | sort account_number | fields firstname, age", + TEST_INDEX_ACCOUNT)); JSONObject resultWithEmpty = executeQuery( String.format( - "source=%s | | where age > 30 | fields firstname, age", TEST_INDEX_ACCOUNT)); + "source=%s | | where age > 30 | sort account_number | fields firstname, age", + TEST_INDEX_ACCOUNT)); assertTrue(resultNormal.similar(resultWithEmpty)); } @@ -136,12 +146,12 @@ public void testMultipleEmptyPipes() throws IOException { JSONObject resultNormal = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age | sort age", + "source=%s | where age > 30 | sort age, account_number | fields firstname, age", TEST_INDEX_ACCOUNT)); JSONObject resultWithEmpty = executeQuery( String.format( - "source=%s | | where age > 30 | | fields firstname, age | sort age", + "source=%s | | where age > 30 | | sort age, account_number | fields firstname, age", TEST_INDEX_ACCOUNT)); assertTrue(resultNormal.similar(resultWithEmpty)); @@ -159,12 +169,12 @@ public void testEmptyPipesAndTrailingPipe() throws IOException { JSONObject resultNormal = executeQuery( String.format( - "source=%s | where age > 30 | fields firstname, age | sort age", + "source=%s | where age > 30 | sort age, account_number | fields firstname, age", TEST_INDEX_ACCOUNT)); JSONObject resultWithEmpty = executeQuery( String.format( - "source=%s | | where age > 30 | fields firstname, age | sort age |", + "source=%s | | where age > 30 | sort age, account_number | fields firstname, age |", TEST_INDEX_ACCOUNT)); assertTrue(resultNormal.similar(resultWithEmpty)); diff --git a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java index a9e36e9b470..626a7658dab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/AnalyticsEngineSecurityIT.java @@ -23,6 +23,8 @@ public class AnalyticsEngineSecurityIT extends SecurityTestBase { private static final String TEST_INDEX = "analytics_security_test"; private static final String FORBIDDEN_INDEX = "analytics_forbidden_test"; + private static final String TEST_INDEX_2 = "analytics_security_extra"; + private static final String TEST_ALIAS = "analytics_alias"; private static final String ALLOWED_USER = "analytics_allowed_user"; private static final String ALLOWED_ROLE = "analytics_allowed_role"; @@ -32,6 +34,10 @@ public class AnalyticsEngineSecurityIT extends SecurityTestBase { private static final String SEARCH_ONLY_ROLE = "analytics_search_only_role"; private static final String WILDCARD_USER = "analytics_wildcard_user"; private static final String WILDCARD_ROLE = "analytics_wildcard_role"; + private static final String ALIAS_USER = "analytics_alias_user"; + private static final String ALIAS_ROLE = "analytics_alias_role"; + private static final String EXACT_PERM_USER = "analytics_exact_perm_user"; + private static final String EXACT_PERM_ROLE = "analytics_exact_perm_role"; private static boolean initialized = false; @@ -73,30 +79,39 @@ private void createTestIndices() throws IOException { // Create composite (analytics-engine-backed) indices so the SQL plugin routes // queries through the analytics engine's DefaultPlanExecutor. createCompositeIndex(TEST_INDEX); + createCompositeIndex(TEST_INDEX_2); + createCompositeIndex(FORBIDDEN_INDEX); + + RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); + opts.addHeader("Content-Type", "application/x-ndjson"); + Request bulk = new Request("POST", "/_bulk"); bulk.addParameter("refresh", "true"); bulk.setJsonEntity( String.format( Locale.ROOT, "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"alice\", \"age\": 30}\n" - + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"bob\", \"age\": 25}\n", + + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"bob\", \"age\": 25}\n" + + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"carol\", \"age\": 28}\n" + + "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"secret\", \"age\": 99}\n", TEST_INDEX, - TEST_INDEX)); - RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder(); - opts.addHeader("Content-Type", "application/x-ndjson"); + TEST_INDEX, + TEST_INDEX_2, + FORBIDDEN_INDEX)); bulk.setOptions(opts); client().performRequest(bulk); - createCompositeIndex(FORBIDDEN_INDEX); - Request bulkF = new Request("POST", "/_bulk"); - bulkF.addParameter("refresh", "true"); - bulkF.setJsonEntity( + // Create alias pointing to TEST_INDEX + Request aliasReq = new Request("POST", "/_aliases"); + aliasReq.setJsonEntity( String.format( Locale.ROOT, - "{\"index\": {\"_index\": \"%s\"}}\n{\"name\": \"secret\", \"age\": 99}\n", - FORBIDDEN_INDEX)); - bulkF.setOptions(opts); - client().performRequest(bulkF); + """ + {"actions": [{"add": {"index": "%s", "alias": "%s"}}]} + """, + TEST_INDEX, + TEST_ALIAS)); + client().performRequest(aliasReq); } private void createCompositeIndex(String index) throws IOException { @@ -166,6 +181,30 @@ private void createSecurityRolesAndUsers() throws IOException { "indices:data/read*", "indices:admin/mappings/get", "indices:monitor/settings/get" }); createUser(WILDCARD_USER, WILDCARD_ROLE); + + // Role with access only to the alias — verifies security plugin resolves alias to + // concrete index and permits access when role's index_patterns matches the alias name. + createRoleWithPermissions( + ALIAS_ROLE, + TEST_ALIAS, + new String[] {"cluster:admin/opensearch/ppl", "cluster:admin/opensearch/sql"}, + new String[] { + "indices:data/read*", "indices:admin/mappings/get", "indices:monitor/settings/get" + }); + createUser(ALIAS_USER, ALIAS_ROLE); + + // Role with exactly indices:data/read/analytics/query — proves this specific permission + // is both necessary and sufficient for analytics engine queries. + createRoleWithPermissions( + EXACT_PERM_ROLE, + TEST_INDEX, + new String[] {"cluster:admin/opensearch/ppl", "cluster:admin/opensearch/sql"}, + new String[] { + "indices:data/read/analytics/query", + "indices:admin/mappings/get", + "indices:monitor/settings/get" + }); + createUser(EXACT_PERM_USER, EXACT_PERM_ROLE); } @Test @@ -216,6 +255,26 @@ public void testPPLQueryDeniedWithSearchPermissionOnly() throws IOException { executePPLAsUser( "source = " + TEST_INDEX + " | fields name, age", SEARCH_ONLY_USER)); assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + String body = org.opensearch.sql.legacy.TestUtils.getResponseBody(e.getResponse(), true); + assertTrue( + "Expected response to reference the missing analytics/query action, got: " + body, + body.contains("indices:data/read/analytics/query")); + } + + @Test + public void testPPLQueryAllowedWithExactAnalyticsQueryPermission() throws IOException { + // User has exactly indices:data/read/analytics/query (not a broad wildcard). + // Proves this specific permission is sufficient for analytics engine queries. + try { + JSONObject result = + executePPLAsUser("source = " + TEST_INDEX + " | fields name, age", EXACT_PERM_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for user with exact analytics/query permission", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } } @Test @@ -247,6 +306,93 @@ public void testPPLQueryDeniedWithWildcardPermissionOnNonMatchingIndex() throws assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); } + // --- Alias-based access tests --- + + @Test + public void testPPLQueryAllowedViaAlias() throws IOException { + // User's role has index_patterns: ["analytics_alias"]. Security plugin resolves the + // alias to the concrete index. Since AnalyticsQueryRequest uses strictExpandOpen(), + // IndexNameExpressionResolver resolves the alias and security matches it against the + // role's index_patterns which includes the alias name. + try { + JSONObject result = + executePPLAsUser("source = " + TEST_ALIAS + " | fields name, age", ALIAS_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for alias-permitted user", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + @Test + public void testPPLQueryDeniedViaAliasForUnauthorizedUser() throws IOException { + // DENIED_USER has no access to analytics_alias or the underlying index. + ResponseException e = + assertThrows( + ResponseException.class, + () -> executePPLAsUser("source = " + TEST_ALIAS + " | fields name, age", DENIED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLQueryAllowedViaConcreteIndexForAliasUser() throws IOException { + // ALIAS_USER's role has index_patterns: ["analytics_alias"]. In OpenSearch's security + // model, granting access to an alias also implicitly grants access to the underlying + // concrete index. This verifies the query succeeds via the concrete name. + try { + JSONObject result = + executePPLAsUser("source = " + TEST_INDEX + " | fields name, age", ALIAS_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for alias user querying concrete index", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + // --- Wildcard index pattern in query tests --- + + @Test + public void testPPLQueryWithWildcardIndexAllowed() throws IOException { + // WILDCARD_USER has index_patterns: ["analytics_security*"]. Query uses wildcard + // "analytics_security*" which resolves to analytics_security_test and + // analytics_security_extra — both match the role's pattern. + try { + JSONObject result = + executePPLAsUser("source = analytics_security* | fields name, age", WILDCARD_USER); + assertTrue("Expected datarows in response", result.has("datarows")); + } catch (ResponseException e) { + assertNotEquals( + "Expected auth to pass (not 403) for wildcard query with matching permissions", + 403, + e.getResponse().getStatusLine().getStatusCode()); + } + } + + @Test + public void testPPLQueryWithWildcardIndexDenied() throws IOException { + // DENIED_USER has no access to any analytics_* indices. + ResponseException e = + assertThrows( + ResponseException.class, + () -> executePPLAsUser("source = analytics_security* | fields name, age", DENIED_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + + @Test + public void testPPLQueryWithWildcardIndexPartialAccessDenied() throws IOException { + // ALIAS_USER only has access to "analytics_alias" — not to "analytics_security*". + // A wildcard query expanding to indices the user lacks permission for should be denied. + ResponseException e = + assertThrows( + ResponseException.class, + () -> executePPLAsUser("source = analytics_security* | fields name, age", ALIAS_USER)); + assertEquals(403, e.getResponse().getStatusLine().getStatusCode()); + } + @Test public void testSQLQueryAllowedForAuthorizedUser() throws IOException { try { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java index d385b54dff1..933fc7b3ed5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeComparisonIT.java @@ -35,6 +35,7 @@ public class DateTimeComparisonIT extends SQLIntegTestCase { @Override public void init() throws Exception { super.init(); + loadIndex(Index.DATETIME_SIMPLE); } private final TimeZone testTz = TimeZone.getDefault(); @@ -237,7 +238,7 @@ public static Iterable compareLtTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') < DATE('1961-04-12')", "ts_d_f", false), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2020-09-16 10:20:30') < TIME('09:07:00')", "ts_t_t", true), - $("TIME('09:07:00') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('09:07:00') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') < TIME('10:20:30')", "ts_t_f", false), $("TIME('20:50:40') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -246,13 +247,13 @@ public static Iterable compareLtTimestampWithOtherTypes() { public static Iterable compareLtDateWithOtherTypes() { return Arrays.asList( $$( - $("DATE('2020-09-16') < TIMESTAMP('3077-04-12 09:07:00')", "d_ts_t", true), + $("DATE('2020-09-16') < TIMESTAMP('2242-04-12 09:07:00')", "d_ts_t", true), $("TIMESTAMP('1961-04-12 09:07:00') < DATE('1984-12-15')", "ts_d_t", true), $("DATE('2020-09-16') < TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') < DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') < TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') < TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') < DATE('2020-09-16')", "t_d_f", false))); } @@ -261,14 +262,14 @@ public static Iterable compareLtTimeWithOtherTypes() { var today = LocalDate.now().toString(); return Arrays.asList( $$( - $("TIME('22:15:07') < TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), + $("TIME('22:15:07') < TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), $("TIMESTAMP('1984-12-15 10:20:30') < TIME('10:20:30')", "ts_t_t", true), $("TIME('10:20:30') < TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') < TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') < DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') < DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') < TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') < DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') < TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') < TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -280,10 +281,10 @@ public static Iterable compareGtTimestampWithOtherTypes() { $("DATE('2020-09-16') > TIMESTAMP('2020-09-15 22:15:07')", "d_ts_t", true), $("TIMESTAMP('2020-09-16 10:20:30') > DATE('2077-04-12')", "ts_d_f", false), $("DATE('1961-04-12') > TIMESTAMP('1961-04-12 00:00:00')", "d_ts_f", false), - $("TIMESTAMP('3077-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), + $("TIMESTAMP('2242-07-08 20:20:30') > TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 10:20:30') > TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') > TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') > TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -294,10 +295,10 @@ public static Iterable compareGtDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') > DATE('2020-09-16')", "ts_d_t", true), $("DATE('2020-09-16') > TIMESTAMP('2020-09-16 00:00:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') > DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') > TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') > TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') > DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -310,8 +311,8 @@ public static Iterable compareGtTimeWithOtherTypes() { $("TIME('10:20:30') > TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') > TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') > DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') > TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') > DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') > TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') > DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') > TIME('09:07:00')", "d_t_f", false))); } @@ -325,8 +326,8 @@ public static Iterable compareLteTimestampWithOtherTypes() { $("TIMESTAMP('2020-09-16 10:20:30') <= DATE('1961-04-12')", "ts_d_f", false), $("DATE('2077-04-12') <= TIMESTAMP('1984-12-15 22:15:07')", "d_ts_f", false), $("TIMESTAMP('" + today + " 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), - $("TIME('09:07:00') <= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_t", true), - $("TIMESTAMP('3077-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), + $("TIME('09:07:00') <= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_t", true), + $("TIMESTAMP('2242-09-16 10:20:30') <= TIME('09:07:00')", "ts_t_f", false), $("TIME('20:50:40') <= TIMESTAMP('" + today + " 10:20:30')", "t_ts_f", false))); } @@ -339,8 +340,8 @@ public static Iterable compareLteDateWithOtherTypes() { $("DATE('2020-09-16') <= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('2077-04-12 09:07:00') <= DATE('2020-09-16')", "ts_d_f", false), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), - $("DATE('3077-04-12') <= TIME('00:00:00')", "d_t_f", false), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), + $("DATE('2242-04-12') <= TIME('00:00:00')", "d_t_f", false), $("TIME('00:00:00') <= DATE('2020-09-16')", "t_d_f", false))); } @@ -353,10 +354,10 @@ public static Iterable compareLteTimeWithOtherTypes() { $("TIMESTAMP('1984-12-15 10:20:30') <= TIME('10:20:30')", "ts_t_t", true), $("TIME('22:15:07') <= TIMESTAMP('1984-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('" + today + " 20:50:42') <= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') <= DATE('3077-04-12')", "t_d_t", true), + $("TIME('09:07:00') <= DATE('2242-04-12')", "t_d_t", true), $("DATE('2020-09-16') <= TIME('09:07:00')", "d_t_t", true), $("TIME('00:00:00') <= DATE('1961-04-12')", "t_d_f", false), - $("DATE('3077-04-12') <= TIME('10:20:30')", "d_t_f", false))); + $("DATE('2242-04-12') <= TIME('10:20:30')", "d_t_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -371,7 +372,7 @@ public static Iterable compareGteTimestampWithOtherTypes() { $("TIMESTAMP('" + today + " 10:20:30') >= TIME('10:20:30')", "ts_t_t", true), $("TIME('20:50:40') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('1977-07-08 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), - $("TIME('09:07:00') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false))); + $("TIME('09:07:00') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -382,10 +383,10 @@ public static Iterable compareGteDateWithOtherTypes() { $("TIMESTAMP('2077-04-12 09:07:00') >= DATE('2020-09-16')", "ts_d_t", true), $("DATE('1961-04-12') >= TIMESTAMP('1961-04-12 09:07:00')", "d_ts_f", false), $("TIMESTAMP('1961-04-12 09:07:00') >= DATE('1984-12-15')", "ts_d_f", false), - $("DATE('3077-04-12') >= TIME('00:00:00')", "d_t_t", true), + $("DATE('2242-04-12') >= TIME('00:00:00')", "d_t_t", true), $("TIME('00:00:00') >= DATE('2020-09-16')", "t_d_t", true), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false))); + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false))); } @ParametersFactory(argumentFormatting = "%1$s => %3$s") @@ -395,11 +396,11 @@ public static Iterable compareGteTimeWithOtherTypes() { $$( $("TIME('10:20:30') >= TIMESTAMP('" + today + " 10:20:30')", "t_ts_t", true), $("TIMESTAMP('" + today + " 20:50:42') >= TIME('10:20:30')", "ts_t_t", true), - $("TIME('22:15:07') >= TIMESTAMP('3077-12-15 22:15:07')", "t_ts_f", false), + $("TIME('22:15:07') >= TIMESTAMP('2242-12-15 22:15:07')", "t_ts_f", false), $("TIMESTAMP('1984-12-15 10:20:30') >= TIME('10:20:30')", "ts_t_f", false), $("TIME('00:00:00') >= DATE('1961-04-12')", "t_d_t", true), - $("DATE('3077-04-12') >= TIME('10:20:30')", "d_t_t", true), - $("TIME('09:07:00') >= DATE('3077-04-12')", "t_d_f", false), + $("DATE('2242-04-12') >= TIME('10:20:30')", "d_t_t", true), + $("TIME('09:07:00') >= DATE('2242-04-12')", "t_d_f", false), $("DATE('2020-09-16') >= TIME('09:07:00')", "d_t_f", false))); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 3373b46303c..71f16f30e43 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -103,6 +103,31 @@ private static void whenExecuteMalformedPayload() throws IOException { client().performRequest(request); } + // An alias field whose path targets a text multi-field (e.g. "source.keyword") must fail with a + // descriptive client error rather than an opaque NullPointerException. + @Test + public void testAliasToKeywordMultiFieldFailsWithBadRequest() throws IOException { + String index = "test_alias_unresolved_keyword"; + createIndexWithMapping( + index, + "{ \"properties\": {" + + " \"source\": { \"type\": \"text\", \"fields\": { \"keyword\": { \"type\":" + + " \"keyword\" } } }," + + " \"source_alias\": { \"type\": \"alias\", \"path\": \"source.keyword\" } } }"); + + expectResponseException() + .hasStatusCode(BAD_REQUEST) + .hasErrorType("ErrorReport") + .containsMessage("Alias field [source_alias] refers to unresolved path [source.keyword]") + .whenExecute(String.format(Locale.ROOT, "SELECT * FROM %s", index)); + } + + private static void createIndexWithMapping(String indexName, String mapping) throws IOException { + Request request = new Request("PUT", "/" + indexName); + request.setJsonEntity(String.format(Locale.ROOT, "{ \"mappings\": %s }", mapping)); + client().performRequest(request); + } + public ResponseExceptionAssertion expectResponseException() { return new ResponseExceptionAssertion(exceptionRule); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index c10b3a219f6..8ae3167b40b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -8,8 +8,10 @@ import static org.hamcrest.Matchers.containsString; import java.io.IOException; +import org.junit.Assume; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; @@ -346,13 +348,22 @@ public void testEfficientModeRejectsScriptPredicate() throws IOException { } // ── k-NN plugin capability check ────────────────────────────────────── - // The default integ-test cluster does not have the k-NN plugin installed. Execution-path + // The plugin's own integ-test cluster does not have the k-NN plugin installed. Execution-path // queries against vectorSearch() should therefore fail with the clear "k-NN plugin missing" // error from KnnPluginCapability, while _explain continues to work because the capability // probe is deferred to scan open() and does not run during analysis/planning. @Test public void testExecutionWithoutKnnPluginReturnsCapabilityError() throws IOException { + // This test asserts the "k-NN plugin not installed" capability error, so it is only meaningful + // on a cluster WITHOUT the k-NN plugin. The OpenSearch distribution bundles opensearch-knn, so + // on the distribution integ-test cluster the query reaches k-NN and fails with a different + // ("not knn_vector type") error instead. Skip there; the missing-plugin path stays covered by + // the plugin's own JVM integ-test cluster, which does not ship k-NN. + Assume.assumeFalse( + "k-NN plugin is installed on this cluster; skipping missing-plugin capability check", + isKnnPluginInstalled()); + ResponseException ex = expectThrows( ResponseException.class, @@ -752,4 +763,16 @@ private void deleteAliasIfExists(String aliasName) { // Alias does not exist, which is fine. } } + + // Mirrors VectorSearchExecutionIT#isKnnPluginInstalled — opensearch-knn is bundled in the + // OpenSearch distribution but not in the plugin's own JVM integ-test cluster. + private static boolean isKnnPluginInstalled() { + try { + Response response = client().performRequest(new Request("GET", "/_cat/plugins?h=component")); + String body = new String(response.getEntity().getContent().readAllBytes()); + return body.contains("opensearch-knn"); + } catch (IOException e) { + return false; + } + } } diff --git a/integ-test/src/test/resources/accounts_extended.json b/integ-test/src/test/resources/accounts_extended.json new file mode 100644 index 00000000000..9af355d797c --- /dev/null +++ b/integ-test/src/test/resources/accounts_extended.json @@ -0,0 +1,2006 @@ +{"index":{"_id":"1"}} +{"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL","_is_real":true} +{"index":{"_id":"6"}} +{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN","_is_real":true} +{"index":{"_id":"13"}} +{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA","_is_real":true} +{"index":{"_id":"18"}} +{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":"Boink","email":"daleadams@boink.com","city":"Orick","state":"MD","_is_real":true} +{"index":{"_id":"20"}} +{"account_number":20,"balance":16418,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place","employer":"Scentric","email":"elinorratliff@scentric.com","city":"Ribera","state":"WA","_is_real":true} +{"index":{"_id":"25"}} +{"account_number":25,"balance":40540,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue","employer":"Filodyne","email":"virginiaayala@filodyne.com","city":"Nicholson","state":"PA","_is_real":true} +{"index":{"_id":"32"}} +{"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street","employer":"Quailcom","email":"dillardmcpherson@quailcom.com","city":"Veguita","state":"IN","_is_real":true} +{"index":{"_id":"37"}} +{"account_number":37,"balance":18612,"firstname":"Mcgee","lastname":"Mooney","age":39,"gender":"M","address":"826 Fillmore Place","employer":"Reversus","email":"mcgeemooney@reversus.com","city":"Tooleville","state":"OK","_is_real":true} +{"index":{"_id":"44"}} +{"account_number":44,"balance":34487,"firstname":"Aurelia","lastname":"Harding","age":37,"gender":"M","address":"502 Baycliff Terrace","employer":"Orbalix","email":"aureliaharding@orbalix.com","city":"Yardville","state":"DE","_is_real":true} +{"index":{"_id":"49"}} +{"account_number":49,"balance":29104,"firstname":"Fulton","lastname":"Holt","age":23,"gender":"F","address":"451 Humboldt Street","employer":"Anocha","email":"fultonholt@anocha.com","city":"Sunriver","state":"RI","_is_real":true} +{"index":{"_id":"51"}} +{"account_number":51,"balance":14097,"firstname":"Burton","lastname":"Meyers","age":31,"gender":"F","address":"334 River Street","employer":"Bezal","email":"burtonmeyers@bezal.com","city":"Jacksonburg","state":"MO","_is_real":true} +{"index":{"_id":"56"}} +{"account_number":56,"balance":14992,"firstname":"Josie","lastname":"Nelson","age":32,"gender":"M","address":"857 Tabor Court","employer":"Emtrac","email":"josienelson@emtrac.com","city":"Sunnyside","state":"UT","_is_real":true} +{"index":{"_id":"63"}} +{"account_number":63,"balance":6077,"firstname":"Hughes","lastname":"Owens","age":30,"gender":"F","address":"510 Sedgwick Street","employer":"Valpreal","email":"hughesowens@valpreal.com","city":"Guilford","state":"KS","_is_real":true} +{"index":{"_id":"68"}} +{"account_number":68,"balance":44214,"firstname":"Hall","lastname":"Key","age":25,"gender":"F","address":"927 Bay Parkway","employer":"Eventex","email":"hallkey@eventex.com","city":"Shawmut","state":"CA","_is_real":true} +{"index":{"_id":"70"}} +{"account_number":70,"balance":38172,"firstname":"Deidre","lastname":"Thompson","age":33,"gender":"F","address":"685 School Lane","employer":"Netplode","email":"deidrethompson@netplode.com","city":"Chestnut","state":"GA","_is_real":true} +{"index":{"_id":"75"}} +{"account_number":75,"balance":40500,"firstname":"Sandoval","lastname":"Kramer","age":22,"gender":"F","address":"166 Irvington Place","employer":"Overfork","email":"sandovalkramer@overfork.com","city":"Limestone","state":"NH","_is_real":true} +{"index":{"_id":"82"}} +{"account_number":82,"balance":41412,"firstname":"Concetta","lastname":"Barnes","age":39,"gender":"F","address":"195 Bayview Place","employer":"Fitcore","email":"concettabarnes@fitcore.com","city":"Summerfield","state":"NC","_is_real":true} +{"index":{"_id":"87"}} +{"account_number":87,"balance":1133,"firstname":"Hewitt","lastname":"Kidd","age":22,"gender":"M","address":"446 Halleck Street","employer":"Isologics","email":"hewittkidd@isologics.com","city":"Coalmont","state":"ME","_is_real":true} +{"index":{"_id":"94"}} +{"account_number":94,"balance":41060,"firstname":"Brittany","lastname":"Cabrera","age":30,"gender":"F","address":"183 Kathleen Court","employer":"Mixers","email":"brittanycabrera@mixers.com","city":"Cornucopia","state":"AZ","_is_real":true} +{"index":{"_id":"99"}} +{"account_number":99,"balance":47159,"firstname":"Ratliff","lastname":"Heath","age":39,"gender":"F","address":"806 Rockwell Place","employer":"Zappix","email":"ratliffheath@zappix.com","city":"Shaft","state":"ND","_is_real":true} +{"index":{"_id":"102"}} +{"account_number":102,"balance":29712,"firstname":"Dena","lastname":"Olson","age":27,"gender":"F","address":"759 Newkirk Avenue","employer":"Hinway","email":"denaolson@hinway.com","city":"Choctaw","state":"NJ","_is_real":true} +{"index":{"_id":"107"}} +{"account_number":107,"balance":48844,"firstname":"Randi","lastname":"Rich","age":28,"gender":"M","address":"694 Jefferson Street","employer":"Netplax","email":"randirich@netplax.com","city":"Bellfountain","state":"SC","_is_real":true} +{"index":{"_id":"114"}} +{"account_number":114,"balance":43045,"firstname":"Josephine","lastname":"Joseph","age":31,"gender":"F","address":"451 Oriental Court","employer":"Turnabout","email":"josephinejoseph@turnabout.com","city":"Sedley","state":"AL","_is_real":true} +{"index":{"_id":"119"}} +{"account_number":119,"balance":49222,"firstname":"Laverne","lastname":"Johnson","age":28,"gender":"F","address":"302 Howard Place","employer":"Senmei","email":"lavernejohnson@senmei.com","city":"Herlong","state":"DC","_is_real":true} +{"index":{"_id":"121"}} +{"account_number":121,"balance":19594,"firstname":"Acevedo","lastname":"Dorsey","age":32,"gender":"M","address":"479 Nova Court","employer":"Netropic","email":"acevedodorsey@netropic.com","city":"Islandia","state":"CT","_is_real":true} +{"index":{"_id":"126"}} +{"account_number":126,"balance":3607,"firstname":"Effie","lastname":"Gates","age":39,"gender":"F","address":"620 National Drive","employer":"Digitalus","email":"effiegates@digitalus.com","city":"Blodgett","state":"MD","_is_real":true} +{"index":{"_id":"133"}} +{"account_number":133,"balance":26135,"firstname":"Deena","lastname":"Richmond","age":36,"gender":"F","address":"646 Underhill Avenue","employer":"Sunclipse","email":"deenarichmond@sunclipse.com","city":"Austinburg","state":"SC","_is_real":true} +{"index":{"_id":"138"}} +{"account_number":138,"balance":9006,"firstname":"Daniel","lastname":"Arnold","age":39,"gender":"F","address":"422 Malbone Street","employer":"Ecstasia","email":"danielarnold@ecstasia.com","city":"Gardiner","state":"MO","_is_real":true} +{"index":{"_id":"140"}} +{"account_number":140,"balance":26696,"firstname":"Cotton","lastname":"Christensen","age":32,"gender":"M","address":"878 Schermerhorn Street","employer":"Prowaste","email":"cottonchristensen@prowaste.com","city":"Mayfair","state":"LA","_is_real":true} +{"index":{"_id":"145"}} +{"account_number":145,"balance":47406,"firstname":"Rowena","lastname":"Wilkinson","age":32,"gender":"M","address":"891 Elton Street","employer":"Asimiline","email":"rowenawilkinson@asimiline.com","city":"Ripley","state":"NH","_is_real":true} +{"index":{"_id":"152"}} +{"account_number":152,"balance":8088,"firstname":"Wolfe","lastname":"Rocha","age":21,"gender":"M","address":"457 Guernsey Street","employer":"Hivedom","email":"wolferocha@hivedom.com","city":"Adelino","state":"MS","_is_real":true} +{"index":{"_id":"157"}} +{"account_number":157,"balance":39868,"firstname":"Claudia","lastname":"Terry","age":20,"gender":"F","address":"132 Gunnison Court","employer":"Lumbrex","email":"claudiaterry@lumbrex.com","city":"Castleton","state":"MD","_is_real":true} +{"index":{"_id":"164"}} +{"account_number":164,"balance":9101,"firstname":"Cummings","lastname":"Little","age":26,"gender":"F","address":"308 Schaefer Street","employer":"Comtrak","email":"cummingslittle@comtrak.com","city":"Chaparrito","state":"WI","_is_real":true} +{"index":{"_id":"169"}} +{"account_number":169,"balance":45953,"firstname":"Hollie","lastname":"Osborn","age":34,"gender":"M","address":"671 Seaview Court","employer":"Musaphics","email":"hollieosborn@musaphics.com","city":"Hanover","state":"GA","_is_real":true} +{"index":{"_id":"171"}} +{"account_number":171,"balance":7091,"firstname":"Nelda","lastname":"Hopper","age":39,"gender":"M","address":"742 Prospect Place","employer":"Equicom","email":"neldahopper@equicom.com","city":"Finderne","state":"SC","_is_real":true} +{"index":{"_id":"176"}} +{"account_number":176,"balance":18607,"firstname":"Kemp","lastname":"Walters","age":28,"gender":"F","address":"906 Howard Avenue","employer":"Eyewax","email":"kempwalters@eyewax.com","city":"Why","state":"KY","_is_real":true} +{"index":{"_id":"183"}} +{"account_number":183,"balance":14223,"firstname":"Hudson","lastname":"English","age":26,"gender":"F","address":"823 Herkimer Place","employer":"Xinware","email":"hudsonenglish@xinware.com","city":"Robbins","state":"ND","_is_real":true} +{"index":{"_id":"188"}} +{"account_number":188,"balance":41504,"firstname":"Tia","lastname":"Miranda","age":24,"gender":"F","address":"583 Ainslie Street","employer":"Jasper","email":"tiamiranda@jasper.com","city":"Summerset","state":"UT","_is_real":true} +{"index":{"_id":"190"}} +{"account_number":190,"balance":3150,"firstname":"Blake","lastname":"Davidson","age":30,"gender":"F","address":"636 Diamond Street","employer":"Quantasis","email":"blakedavidson@quantasis.com","city":"Crumpler","state":"KY","_is_real":true} +{"index":{"_id":"195"}} +{"account_number":195,"balance":5025,"firstname":"Kaye","lastname":"Gibson","age":31,"gender":"M","address":"955 Hopkins Street","employer":"Zork","email":"kayegibson@zork.com","city":"Ola","state":"WY","_is_real":true} +{"index":{"_id":"203"}} +{"account_number":203,"balance":21890,"firstname":"Eve","lastname":"Wyatt","age":33,"gender":"M","address":"435 Furman Street","employer":"Assitia","email":"evewyatt@assitia.com","city":"Jamestown","state":"MN","_is_real":true} +{"index":{"_id":"208"}} +{"account_number":208,"balance":40760,"firstname":"Garcia","lastname":"Hess","age":26,"gender":"F","address":"810 Nostrand Avenue","employer":"Quiltigen","email":"garciahess@quiltigen.com","city":"Brooktrails","state":"GA","_is_real":true} +{"index":{"_id":"210"}} +{"account_number":210,"balance":33946,"firstname":"Cherry","lastname":"Carey","age":24,"gender":"M","address":"539 Tiffany Place","employer":"Martgo","email":"cherrycarey@martgo.com","city":"Fairacres","state":"AK","_is_real":true} +{"index":{"_id":"215"}} +{"account_number":215,"balance":37427,"firstname":"Copeland","lastname":"Solomon","age":20,"gender":"M","address":"741 McDonald Avenue","employer":"Recognia","email":"copelandsolomon@recognia.com","city":"Edmund","state":"ME","_is_real":true} +{"index":{"_id":"222"}} +{"account_number":222,"balance":14764,"firstname":"Rachelle","lastname":"Rice","age":36,"gender":"M","address":"333 Narrows Avenue","employer":"Enaut","email":"rachellerice@enaut.com","city":"Wright","state":"AZ","_is_real":true} +{"index":{"_id":"227"}} +{"account_number":227,"balance":19780,"firstname":"Coleman","lastname":"Berg","age":22,"gender":"M","address":"776 Little Street","employer":"Exoteric","email":"colemanberg@exoteric.com","city":"Eagleville","state":"WV","_is_real":true} +{"index":{"_id":"234"}} +{"account_number":234,"balance":44207,"firstname":"Betty","lastname":"Hall","age":37,"gender":"F","address":"709 Garfield Place","employer":"Miraclis","email":"bettyhall@miraclis.com","city":"Bendon","state":"NY","_is_real":true} +{"index":{"_id":"239"}} +{"account_number":239,"balance":25719,"firstname":"Chang","lastname":"Boyer","age":36,"gender":"M","address":"895 Brigham Street","employer":"Qaboos","email":"changboyer@qaboos.com","city":"Belgreen","state":"NH","_is_real":true} +{"index":{"_id":"241"}} +{"account_number":241,"balance":25379,"firstname":"Schroeder","lastname":"Harrington","age":26,"gender":"M","address":"610 Tapscott Avenue","employer":"Otherway","email":"schroederharrington@otherway.com","city":"Ebro","state":"TX","_is_real":true} +{"index":{"_id":"246"}} +{"account_number":246,"balance":28405,"firstname":"Katheryn","lastname":"Foster","age":21,"gender":"F","address":"259 Kane Street","employer":"Quantalia","email":"katherynfoster@quantalia.com","city":"Bath","state":"TX","_is_real":true} +{"index":{"_id":"253"}} +{"account_number":253,"balance":20240,"firstname":"Melissa","lastname":"Gould","age":31,"gender":"M","address":"440 Fuller Place","employer":"Buzzopia","email":"melissagould@buzzopia.com","city":"Lumberton","state":"MD","_is_real":true} +{"index":{"_id":"258"}} +{"account_number":258,"balance":5712,"firstname":"Lindsey","lastname":"Hawkins","age":37,"gender":"M","address":"706 Frost Street","employer":"Enormo","email":"lindseyhawkins@enormo.com","city":"Gardners","state":"AK","_is_real":true} +{"index":{"_id":"260"}} +{"account_number":260,"balance":2726,"firstname":"Kari","lastname":"Skinner","age":30,"gender":"F","address":"735 Losee Terrace","employer":"Singavera","email":"kariskinner@singavera.com","city":"Rushford","state":"WV","_is_real":true} +{"index":{"_id":"265"}} +{"account_number":265,"balance":46910,"firstname":"Marion","lastname":"Schneider","age":26,"gender":"F","address":"574 Everett Avenue","employer":"Evidends","email":"marionschneider@evidends.com","city":"Maplewood","state":"WY","_is_real":true} +{"index":{"_id":"272"}} +{"account_number":272,"balance":19253,"firstname":"Lilly","lastname":"Morgan","age":25,"gender":"F","address":"689 Fleet Street","employer":"Biolive","email":"lillymorgan@biolive.com","city":"Sunbury","state":"OH","_is_real":true} +{"index":{"_id":"277"}} +{"account_number":277,"balance":29564,"firstname":"Romero","lastname":"Lott","age":31,"gender":"M","address":"456 Danforth Street","employer":"Plasto","email":"romerolott@plasto.com","city":"Vincent","state":"VT","_is_real":true} +{"index":{"_id":"284"}} +{"account_number":284,"balance":22806,"firstname":"Randolph","lastname":"Banks","age":29,"gender":"M","address":"875 Hamilton Avenue","employer":"Caxt","email":"randolphbanks@caxt.com","city":"Crawfordsville","state":"WA","_is_real":true} +{"index":{"_id":"289"}} +{"account_number":289,"balance":7798,"firstname":"Blair","lastname":"Church","age":29,"gender":"M","address":"370 Sutton Street","employer":"Cubix","email":"blairchurch@cubix.com","city":"Nile","state":"NH","_is_real":true} +{"index":{"_id":"291"}} +{"account_number":291,"balance":19955,"firstname":"Lynn","lastname":"Pollard","age":40,"gender":"F","address":"685 Pierrepont Street","employer":"Slambda","email":"lynnpollard@slambda.com","city":"Mappsville","state":"ID","_is_real":true} +{"index":{"_id":"296"}} +{"account_number":296,"balance":24606,"firstname":"Rosa","lastname":"Oliver","age":34,"gender":"M","address":"168 Woodbine Street","employer":"Idetica","email":"rosaoliver@idetica.com","city":"Robinson","state":"WY","_is_real":true} +{"index":{"_id":"304"}} +{"account_number":304,"balance":28647,"firstname":"Palmer","lastname":"Clark","age":35,"gender":"M","address":"866 Boulevard Court","employer":"Maximind","email":"palmerclark@maximind.com","city":"Avalon","state":"NH","_is_real":true} +{"index":{"_id":"309"}} +{"account_number":309,"balance":3830,"firstname":"Rosemarie","lastname":"Nieves","age":30,"gender":"M","address":"206 Alice Court","employer":"Zounds","email":"rosemarienieves@zounds.com","city":"Ferney","state":"AR","_is_real":true} +{"index":{"_id":"311"}} +{"account_number":311,"balance":13388,"firstname":"Vinson","lastname":"Ballard","age":23,"gender":"F","address":"960 Glendale Court","employer":"Gynk","email":"vinsonballard@gynk.com","city":"Fairforest","state":"WY","_is_real":true} +{"index":{"_id":"316"}} +{"account_number":316,"balance":8214,"firstname":"Anita","lastname":"Ewing","age":32,"gender":"M","address":"396 Lombardy Street","employer":"Panzent","email":"anitaewing@panzent.com","city":"Neahkahnie","state":"WY","_is_real":true} +{"index":{"_id":"323"}} +{"account_number":323,"balance":42230,"firstname":"Chelsea","lastname":"Gamble","age":34,"gender":"F","address":"356 Dare Court","employer":"Isosphere","email":"chelseagamble@isosphere.com","city":"Dundee","state":"MD","_is_real":true} +{"index":{"_id":"328"}} +{"account_number":328,"balance":12523,"firstname":"Good","lastname":"Campbell","age":27,"gender":"F","address":"438 Hicks Street","employer":"Gracker","email":"goodcampbell@gracker.com","city":"Marion","state":"CA","_is_real":true} +{"index":{"_id":"330"}} +{"account_number":330,"balance":41620,"firstname":"Yvette","lastname":"Browning","age":34,"gender":"F","address":"431 Beekman Place","employer":"Marketoid","email":"yvettebrowning@marketoid.com","city":"Talpa","state":"CO","_is_real":true} +{"index":{"_id":"335"}} +{"account_number":335,"balance":35433,"firstname":"Vera","lastname":"Hansen","age":24,"gender":"M","address":"252 Bushwick Avenue","employer":"Zanilla","email":"verahansen@zanilla.com","city":"Manila","state":"TN","_is_real":true} +{"index":{"_id":"342"}} +{"account_number":342,"balance":33670,"firstname":"Vivian","lastname":"Wells","age":36,"gender":"M","address":"570 Cobek Court","employer":"Nutralab","email":"vivianwells@nutralab.com","city":"Fontanelle","state":"OK","_is_real":true} +{"index":{"_id":"347"}} +{"account_number":347,"balance":36038,"firstname":"Gould","lastname":"Carson","age":24,"gender":"F","address":"784 Pulaski Street","employer":"Mobildata","email":"gouldcarson@mobildata.com","city":"Goochland","state":"MI","_is_real":true} +{"index":{"_id":"354"}} +{"account_number":354,"balance":21294,"firstname":"Kidd","lastname":"Mclean","age":22,"gender":"M","address":"691 Saratoga Avenue","employer":"Ronbert","email":"kiddmclean@ronbert.com","city":"Tioga","state":"ME","_is_real":true} +{"index":{"_id":"359"}} +{"account_number":359,"balance":29927,"firstname":"Vanessa","lastname":"Harvey","age":28,"gender":"F","address":"679 Rutledge Street","employer":"Zentime","email":"vanessaharvey@zentime.com","city":"Williston","state":"IL","_is_real":true} +{"index":{"_id":"361"}} +{"account_number":361,"balance":23659,"firstname":"Noreen","lastname":"Shelton","age":36,"gender":"M","address":"702 Tillary Street","employer":"Medmex","email":"noreenshelton@medmex.com","city":"Derwood","state":"NH","_is_real":true} +{"index":{"_id":"366"}} +{"account_number":366,"balance":42368,"firstname":"Lydia","lastname":"Cooke","age":31,"gender":"M","address":"470 Coleman Street","employer":"Comstar","email":"lydiacooke@comstar.com","city":"Datil","state":"TN","_is_real":true} +{"index":{"_id":"373"}} +{"account_number":373,"balance":9671,"firstname":"Simpson","lastname":"Carpenter","age":21,"gender":"M","address":"837 Horace Court","employer":"Snips","email":"simpsoncarpenter@snips.com","city":"Tolu","state":"MA","_is_real":true} +{"index":{"_id":"378"}} +{"account_number":378,"balance":27100,"firstname":"Watson","lastname":"Simpson","age":36,"gender":"F","address":"644 Thomas Street","employer":"Wrapture","email":"watsonsimpson@wrapture.com","city":"Keller","state":"TX","_is_real":true} +{"index":{"_id":"380"}} +{"account_number":380,"balance":35628,"firstname":"Fernandez","lastname":"Reid","age":33,"gender":"F","address":"154 Melba Court","employer":"Cosmosis","email":"fernandezreid@cosmosis.com","city":"Boyd","state":"NE","_is_real":true} +{"index":{"_id":"385"}} +{"account_number":385,"balance":11022,"firstname":"Rosalinda","lastname":"Valencia","age":22,"gender":"M","address":"933 Lloyd Street","employer":"Zoarere","email":"rosalindavalencia@zoarere.com","city":"Waverly","state":"GA","_is_real":true} +{"index":{"_id":"392"}} +{"account_number":392,"balance":31613,"firstname":"Dotson","lastname":"Dean","age":35,"gender":"M","address":"136 Ford Street","employer":"Petigems","email":"dotsondean@petigems.com","city":"Chical","state":"SD","_is_real":true} +{"index":{"_id":"397"}} +{"account_number":397,"balance":37418,"firstname":"Leonard","lastname":"Gray","age":36,"gender":"F","address":"840 Morgan Avenue","employer":"Recritube","email":"leonardgray@recritube.com","city":"Edenburg","state":"AL","_is_real":true} +{"index":{"_id":"400"}} +{"account_number":400,"balance":20685,"firstname":"Kane","lastname":"King","age":21,"gender":"F","address":"405 Cornelia Street","employer":"Tri@Tribalog","email":"kaneking@tri@tribalog.com","city":"Gulf","state":"VT","_is_real":true} +{"index":{"_id":"405"}} +{"account_number":405,"balance":5679,"firstname":"Strickland","lastname":"Fuller","age":26,"gender":"M","address":"990 Concord Street","employer":"Digique","email":"stricklandfuller@digique.com","city":"Southmont","state":"NV","_is_real":true} +{"index":{"_id":"412"}} +{"account_number":412,"balance":27436,"firstname":"Ilene","lastname":"Abbott","age":26,"gender":"M","address":"846 Vine Street","employer":"Typhonica","email":"ileneabbott@typhonica.com","city":"Cedarville","state":"VT","_is_real":true} +{"index":{"_id":"417"}} +{"account_number":417,"balance":1788,"firstname":"Wheeler","lastname":"Ayers","age":35,"gender":"F","address":"677 Hope Street","employer":"Fortean","email":"wheelerayers@fortean.com","city":"Ironton","state":"PA","_is_real":true} +{"index":{"_id":"424"}} +{"account_number":424,"balance":36818,"firstname":"Tracie","lastname":"Gregory","age":34,"gender":"M","address":"112 Hunterfly Place","employer":"Comstruct","email":"traciegregory@comstruct.com","city":"Onton","state":"TN","_is_real":true} +{"index":{"_id":"429"}} +{"account_number":429,"balance":46970,"firstname":"Cantu","lastname":"Lindsey","age":31,"gender":"M","address":"404 Willoughby Avenue","employer":"Inquala","email":"cantulindsey@inquala.com","city":"Cowiche","state":"IA","_is_real":true} +{"index":{"_id":"431"}} +{"account_number":431,"balance":13136,"firstname":"Laurie","lastname":"Shaw","age":26,"gender":"F","address":"263 Aviation Road","employer":"Zillanet","email":"laurieshaw@zillanet.com","city":"Harmon","state":"WV","_is_real":true} +{"index":{"_id":"436"}} +{"account_number":436,"balance":27585,"firstname":"Alexander","lastname":"Sargent","age":23,"gender":"M","address":"363 Albemarle Road","employer":"Fangold","email":"alexandersargent@fangold.com","city":"Calpine","state":"OR","_is_real":true} +{"index":{"_id":"443"}} +{"account_number":443,"balance":7588,"firstname":"Huff","lastname":"Thomas","age":23,"gender":"M","address":"538 Erskine Loop","employer":"Accufarm","email":"huffthomas@accufarm.com","city":"Corinne","state":"AL","_is_real":true} +{"index":{"_id":"448"}} +{"account_number":448,"balance":22776,"firstname":"Adriana","lastname":"Mcfadden","age":35,"gender":"F","address":"984 Woodside Avenue","employer":"Telequiet","email":"adrianamcfadden@telequiet.com","city":"Darrtown","state":"WI","_is_real":true} +{"index":{"_id":"450"}} +{"account_number":450,"balance":2643,"firstname":"Bradford","lastname":"Nielsen","age":25,"gender":"M","address":"487 Keen Court","employer":"Exovent","email":"bradfordnielsen@exovent.com","city":"Hamilton","state":"DE","_is_real":true} +{"index":{"_id":"455"}} +{"account_number":455,"balance":39556,"firstname":"Lynn","lastname":"Tran","age":36,"gender":"M","address":"741 Richmond Street","employer":"Optyk","email":"lynntran@optyk.com","city":"Clinton","state":"WV","_is_real":true} +{"index":{"_id":"462"}} +{"account_number":462,"balance":10871,"firstname":"Calderon","lastname":"Day","age":27,"gender":"M","address":"810 Milford Street","employer":"Cofine","email":"calderonday@cofine.com","city":"Kula","state":"OK","_is_real":true} +{"index":{"_id":"467"}} +{"account_number":467,"balance":6312,"firstname":"Angelica","lastname":"May","age":32,"gender":"F","address":"384 Karweg Place","employer":"Keeg","email":"angelicamay@keeg.com","city":"Tetherow","state":"IA","_is_real":true} +{"index":{"_id":"474"}} +{"account_number":474,"balance":35896,"firstname":"Obrien","lastname":"Walton","age":40,"gender":"F","address":"192 Ide Court","employer":"Suremax","email":"obrienwalton@suremax.com","city":"Crucible","state":"UT","_is_real":true} +{"index":{"_id":"479"}} +{"account_number":479,"balance":31865,"firstname":"Cameron","lastname":"Ross","age":40,"gender":"M","address":"904 Bouck Court","employer":"Telpod","email":"cameronross@telpod.com","city":"Nord","state":"MO","_is_real":true} +{"index":{"_id":"481"}} +{"account_number":481,"balance":20024,"firstname":"Lina","lastname":"Stanley","age":33,"gender":"M","address":"361 Hanover Place","employer":"Strozen","email":"linastanley@strozen.com","city":"Wyoming","state":"NC","_is_real":true} +{"index":{"_id":"486"}} +{"account_number":486,"balance":35902,"firstname":"Dixie","lastname":"Fuentes","age":22,"gender":"F","address":"991 Applegate Court","employer":"Portico","email":"dixiefuentes@portico.com","city":"Salix","state":"VA","_is_real":true} +{"index":{"_id":"493"}} +{"account_number":493,"balance":5871,"firstname":"Campbell","lastname":"Best","age":24,"gender":"M","address":"297 Friel Place","employer":"Fanfare","email":"campbellbest@fanfare.com","city":"Kidder","state":"GA","_is_real":true} +{"index":{"_id":"498"}} +{"account_number":498,"balance":10516,"firstname":"Stella","lastname":"Hinton","age":39,"gender":"F","address":"649 Columbia Place","employer":"Flyboyz","email":"stellahinton@flyboyz.com","city":"Crenshaw","state":"SC","_is_real":true} +{"index":{"_id":"501"}} +{"account_number":501,"balance":16572,"firstname":"Kelley","lastname":"Ochoa","age":36,"gender":"M","address":"451 Clifton Place","employer":"Bluplanet","email":"kelleyochoa@bluplanet.com","city":"Gouglersville","state":"CT","_is_real":true} +{"index":{"_id":"506"}} +{"account_number":506,"balance":43440,"firstname":"Davidson","lastname":"Salas","age":28,"gender":"M","address":"731 Cleveland Street","employer":"Sequitur","email":"davidsonsalas@sequitur.com","city":"Lloyd","state":"ME","_is_real":true} +{"index":{"_id":"513"}} +{"account_number":513,"balance":30040,"firstname":"Maryellen","lastname":"Rose","age":37,"gender":"F","address":"428 Durland Place","employer":"Waterbaby","email":"maryellenrose@waterbaby.com","city":"Kiskimere","state":"RI","_is_real":true} +{"index":{"_id":"518"}} +{"account_number":518,"balance":48954,"firstname":"Finch","lastname":"Curtis","age":29,"gender":"F","address":"137 Ryder Street","employer":"Viagrand","email":"finchcurtis@viagrand.com","city":"Riverton","state":"MO","_is_real":true} +{"index":{"_id":"520"}} +{"account_number":520,"balance":27987,"firstname":"Brandy","lastname":"Calhoun","age":32,"gender":"M","address":"818 Harden Street","employer":"Maxemia","email":"brandycalhoun@maxemia.com","city":"Sidman","state":"OR","_is_real":true} +{"index":{"_id":"525"}} +{"account_number":525,"balance":23545,"firstname":"Holly","lastname":"Miles","age":25,"gender":"M","address":"746 Ludlam Place","employer":"Xurban","email":"hollymiles@xurban.com","city":"Harold","state":"AR","_is_real":true} +{"index":{"_id":"532"}} +{"account_number":532,"balance":17207,"firstname":"Hardin","lastname":"Kirk","age":26,"gender":"M","address":"268 Canarsie Road","employer":"Exposa","email":"hardinkirk@exposa.com","city":"Stouchsburg","state":"IL","_is_real":true} +{"index":{"_id":"537"}} +{"account_number":537,"balance":31069,"firstname":"Morin","lastname":"Frost","age":29,"gender":"M","address":"910 Lake Street","employer":"Primordia","email":"morinfrost@primordia.com","city":"Rivera","state":"DE","_is_real":true} +{"index":{"_id":"544"}} +{"account_number":544,"balance":41735,"firstname":"Short","lastname":"Dennis","age":21,"gender":"F","address":"908 Glen Street","employer":"Minga","email":"shortdennis@minga.com","city":"Dale","state":"KY","_is_real":true} +{"index":{"_id":"549"}} +{"account_number":549,"balance":1932,"firstname":"Jacqueline","lastname":"Maxwell","age":40,"gender":"M","address":"444 Schenck Place","employer":"Fuelworks","email":"jacquelinemaxwell@fuelworks.com","city":"Oretta","state":"OR","_is_real":true} +{"index":{"_id":"551"}} +{"account_number":551,"balance":21732,"firstname":"Milagros","lastname":"Travis","age":27,"gender":"F","address":"380 Murdock Court","employer":"Sloganaut","email":"milagrostravis@sloganaut.com","city":"Homeland","state":"AR","_is_real":true} +{"index":{"_id":"556"}} +{"account_number":556,"balance":36420,"firstname":"Collier","lastname":"Odonnell","age":35,"gender":"M","address":"591 Nolans Lane","employer":"Sultraxin","email":"collierodonnell@sultraxin.com","city":"Fulford","state":"MD","_is_real":true} +{"index":{"_id":"563"}} +{"account_number":563,"balance":43403,"firstname":"Morgan","lastname":"Torres","age":30,"gender":"F","address":"672 Belvidere Street","employer":"Quonata","email":"morgantorres@quonata.com","city":"Hollymead","state":"KY","_is_real":true} +{"index":{"_id":"568"}} +{"account_number":568,"balance":36628,"firstname":"Lesa","lastname":"Maynard","age":29,"gender":"F","address":"295 Whitty Lane","employer":"Coash","email":"lesamaynard@coash.com","city":"Broadlands","state":"VT","_is_real":true} +{"index":{"_id":"570"}} +{"account_number":570,"balance":26751,"firstname":"Church","lastname":"Mercado","age":24,"gender":"F","address":"892 Wyckoff Street","employer":"Xymonk","email":"churchmercado@xymonk.com","city":"Gloucester","state":"KY","_is_real":true} +{"index":{"_id":"575"}} +{"account_number":575,"balance":12588,"firstname":"Buchanan","lastname":"Pope","age":39,"gender":"M","address":"581 Sumner Place","employer":"Stucco","email":"buchananpope@stucco.com","city":"Ellerslie","state":"MD","_is_real":true} +{"index":{"_id":"582"}} +{"account_number":582,"balance":33371,"firstname":"Manning","lastname":"Guthrie","age":24,"gender":"F","address":"271 Jodie Court","employer":"Xerex","email":"manningguthrie@xerex.com","city":"Breinigsville","state":"NM","_is_real":true} +{"index":{"_id":"587"}} +{"account_number":587,"balance":3468,"firstname":"Carly","lastname":"Johns","age":33,"gender":"M","address":"390 Noll Street","employer":"Gallaxia","email":"carlyjohns@gallaxia.com","city":"Emison","state":"DC","_is_real":true} +{"index":{"_id":"594"}} +{"account_number":594,"balance":28194,"firstname":"Golden","lastname":"Donovan","age":26,"gender":"M","address":"199 Jewel Street","employer":"Organica","email":"goldendonovan@organica.com","city":"Macdona","state":"RI","_is_real":true} +{"index":{"_id":"599"}} +{"account_number":599,"balance":11944,"firstname":"Joanna","lastname":"Jennings","age":36,"gender":"F","address":"318 Irving Street","employer":"Extremo","email":"joannajennings@extremo.com","city":"Bartley","state":"MI","_is_real":true} +{"index":{"_id":"602"}} +{"account_number":602,"balance":38699,"firstname":"Mcgowan","lastname":"Mcclain","age":33,"gender":"M","address":"361 Stoddard Place","employer":"Oatfarm","email":"mcgowanmcclain@oatfarm.com","city":"Kapowsin","state":"MI","_is_real":true} +{"index":{"_id":"607"}} +{"account_number":607,"balance":38350,"firstname":"White","lastname":"Small","age":38,"gender":"F","address":"736 Judge Street","employer":"Immunics","email":"whitesmall@immunics.com","city":"Fairfield","state":"HI","_is_real":true} +{"index":{"_id":"614"}} +{"account_number":614,"balance":13157,"firstname":"Salazar","lastname":"Howard","age":35,"gender":"F","address":"847 Imlay Street","employer":"Retrack","email":"salazarhoward@retrack.com","city":"Grill","state":"FL","_is_real":true} +{"index":{"_id":"619"}} +{"account_number":619,"balance":48755,"firstname":"Grimes","lastname":"Reynolds","age":36,"gender":"M","address":"378 Denton Place","employer":"Frenex","email":"grimesreynolds@frenex.com","city":"Murillo","state":"LA","_is_real":true} +{"index":{"_id":"621"}} +{"account_number":621,"balance":35480,"firstname":"Leslie","lastname":"Sloan","age":26,"gender":"F","address":"336 Kansas Place","employer":"Dancity","email":"lesliesloan@dancity.com","city":"Corriganville","state":"AR","_is_real":true} +{"index":{"_id":"626"}} +{"account_number":626,"balance":19498,"firstname":"Ava","lastname":"Richardson","age":31,"gender":"F","address":"666 Nautilus Avenue","employer":"Cinaster","email":"avarichardson@cinaster.com","city":"Sutton","state":"AL","_is_real":true} +{"index":{"_id":"633"}} +{"account_number":633,"balance":35874,"firstname":"Conner","lastname":"Ramos","age":34,"gender":"M","address":"575 Agate Court","employer":"Insource","email":"connerramos@insource.com","city":"Madaket","state":"OK","_is_real":true} +{"index":{"_id":"638"}} +{"account_number":638,"balance":2658,"firstname":"Bridget","lastname":"Gallegos","age":31,"gender":"M","address":"383 Wogan Terrace","employer":"Songlines","email":"bridgetgallegos@songlines.com","city":"Linganore","state":"WA","_is_real":true} +{"index":{"_id":"640"}} +{"account_number":640,"balance":35596,"firstname":"Candace","lastname":"Hancock","age":25,"gender":"M","address":"574 Riverdale Avenue","employer":"Animalia","email":"candacehancock@animalia.com","city":"Blandburg","state":"KY","_is_real":true} +{"index":{"_id":"645"}} +{"account_number":645,"balance":29362,"firstname":"Edwina","lastname":"Hutchinson","age":26,"gender":"F","address":"892 Pacific Street","employer":"Essensia","email":"edwinahutchinson@essensia.com","city":"Dowling","state":"NE","_is_real":true} +{"index":{"_id":"652"}} +{"account_number":652,"balance":17363,"firstname":"Bonner","lastname":"Garner","age":26,"gender":"M","address":"219 Grafton Street","employer":"Utarian","email":"bonnergarner@utarian.com","city":"Vandiver","state":"PA","_is_real":true} +{"index":{"_id":"657"}} +{"account_number":657,"balance":40475,"firstname":"Kathleen","lastname":"Wilder","age":34,"gender":"F","address":"286 Sutter Avenue","employer":"Solgan","email":"kathleenwilder@solgan.com","city":"Graniteville","state":"MI","_is_real":true} +{"index":{"_id":"664"}} +{"account_number":664,"balance":16163,"firstname":"Hart","lastname":"Mccormick","age":40,"gender":"M","address":"144 Guider Avenue","employer":"Dyno","email":"hartmccormick@dyno.com","city":"Carbonville","state":"ID","_is_real":true} +{"index":{"_id":"669"}} +{"account_number":669,"balance":16934,"firstname":"Jewel","lastname":"Estrada","age":28,"gender":"M","address":"896 Meeker Avenue","employer":"Zilla","email":"jewelestrada@zilla.com","city":"Goodville","state":"PA","_is_real":true} +{"index":{"_id":"671"}} +{"account_number":671,"balance":29029,"firstname":"Antoinette","lastname":"Cook","age":34,"gender":"M","address":"375 Cumberland Street","employer":"Harmoney","email":"antoinettecook@harmoney.com","city":"Bergoo","state":"VT","_is_real":true} +{"index":{"_id":"676"}} +{"account_number":676,"balance":23842,"firstname":"Lisa","lastname":"Dudley","age":34,"gender":"M","address":"506 Vanderveer Street","employer":"Tropoli","email":"lisadudley@tropoli.com","city":"Konterra","state":"NY","_is_real":true} +{"index":{"_id":"683"}} +{"account_number":683,"balance":4381,"firstname":"Matilda","lastname":"Berger","age":39,"gender":"M","address":"884 Noble Street","employer":"Fibrodyne","email":"matildaberger@fibrodyne.com","city":"Shepardsville","state":"TN","_is_real":true} +{"index":{"_id":"688"}} +{"account_number":688,"balance":17931,"firstname":"Freeman","lastname":"Zamora","age":22,"gender":"F","address":"114 Herzl Street","employer":"Elemantra","email":"freemanzamora@elemantra.com","city":"Libertytown","state":"NM","_is_real":true} +{"index":{"_id":"690"}} +{"account_number":690,"balance":18127,"firstname":"Russo","lastname":"Swanson","age":35,"gender":"F","address":"256 Roebling Street","employer":"Zaj","email":"russoswanson@zaj.com","city":"Hoagland","state":"MI","_is_real":true} +{"index":{"_id":"695"}} +{"account_number":695,"balance":36800,"firstname":"Gonzales","lastname":"Mcfarland","age":26,"gender":"F","address":"647 Louisa Street","employer":"Songbird","email":"gonzalesmcfarland@songbird.com","city":"Crisman","state":"ID","_is_real":true} +{"index":{"_id":"703"}} +{"account_number":703,"balance":27443,"firstname":"Dona","lastname":"Burton","age":29,"gender":"M","address":"489 Flatlands Avenue","employer":"Cytrex","email":"donaburton@cytrex.com","city":"Reno","state":"VA","_is_real":true} +{"index":{"_id":"708"}} +{"account_number":708,"balance":34002,"firstname":"May","lastname":"Ortiz","age":28,"gender":"F","address":"244 Chauncey Street","employer":"Syntac","email":"mayortiz@syntac.com","city":"Munjor","state":"ID","_is_real":true} +{"index":{"_id":"710"}} +{"account_number":710,"balance":33650,"firstname":"Shelton","lastname":"Stark","age":37,"gender":"M","address":"404 Ovington Avenue","employer":"Kraggle","email":"sheltonstark@kraggle.com","city":"Ogema","state":"TN","_is_real":true} +{"index":{"_id":"715"}} +{"account_number":715,"balance":23734,"firstname":"Tammi","lastname":"Hodge","age":24,"gender":"M","address":"865 Church Lane","employer":"Netur","email":"tammihodge@netur.com","city":"Lacomb","state":"KS","_is_real":true} +{"index":{"_id":"722"}} +{"account_number":722,"balance":27256,"firstname":"Roberts","lastname":"Beasley","age":34,"gender":"F","address":"305 Kings Hwy","employer":"Quintity","email":"robertsbeasley@quintity.com","city":"Hayden","state":"PA","_is_real":true} +{"index":{"_id":"727"}} +{"account_number":727,"balance":27263,"firstname":"Natasha","lastname":"Knapp","age":36,"gender":"M","address":"723 Hubbard Street","employer":"Exostream","email":"natashaknapp@exostream.com","city":"Trexlertown","state":"LA","_is_real":true} +{"index":{"_id":"734"}} +{"account_number":734,"balance":20325,"firstname":"Keri","lastname":"Kinney","age":23,"gender":"M","address":"490 Balfour Place","employer":"Retrotex","email":"kerikinney@retrotex.com","city":"Salunga","state":"PA","_is_real":true} +{"index":{"_id":"739"}} +{"account_number":739,"balance":39063,"firstname":"Gwen","lastname":"Hardy","age":33,"gender":"F","address":"733 Stuart Street","employer":"Exozent","email":"gwenhardy@exozent.com","city":"Drytown","state":"NY","_is_real":true} +{"index":{"_id":"741"}} +{"account_number":741,"balance":33074,"firstname":"Nielsen","lastname":"Good","age":22,"gender":"M","address":"404 Norfolk Street","employer":"Kiggle","email":"nielsengood@kiggle.com","city":"Cumberland","state":"WA","_is_real":true} +{"index":{"_id":"746"}} +{"account_number":746,"balance":15970,"firstname":"Marguerite","lastname":"Wall","age":28,"gender":"F","address":"364 Crosby Avenue","employer":"Aquoavo","email":"margueritewall@aquoavo.com","city":"Jeff","state":"MI","_is_real":true} +{"index":{"_id":"753"}} +{"account_number":753,"balance":33340,"firstname":"Katina","lastname":"Alford","age":21,"gender":"F","address":"690 Ross Street","employer":"Intrawear","email":"katinaalford@intrawear.com","city":"Grimsley","state":"OK","_is_real":true} +{"index":{"_id":"758"}} +{"account_number":758,"balance":15739,"firstname":"Berta","lastname":"Short","age":28,"gender":"M","address":"149 Surf Avenue","employer":"Ozean","email":"bertashort@ozean.com","city":"Odessa","state":"UT","_is_real":true} +{"index":{"_id":"760"}} +{"account_number":760,"balance":40996,"firstname":"Rhea","lastname":"Blair","age":37,"gender":"F","address":"440 Hubbard Place","employer":"Bicol","email":"rheablair@bicol.com","city":"Stockwell","state":"LA","_is_real":true} +{"index":{"_id":"765"}} +{"account_number":765,"balance":31278,"firstname":"Knowles","lastname":"Cunningham","age":23,"gender":"M","address":"753 Macdougal Street","employer":"Thredz","email":"knowlescunningham@thredz.com","city":"Thomasville","state":"WA","_is_real":true} +{"index":{"_id":"772"}} +{"account_number":772,"balance":37849,"firstname":"Eloise","lastname":"Sparks","age":21,"gender":"M","address":"608 Willow Street","employer":"Satiance","email":"eloisesparks@satiance.com","city":"Richford","state":"NY","_is_real":true} +{"index":{"_id":"777"}} +{"account_number":777,"balance":48294,"firstname":"Adkins","lastname":"Mejia","age":32,"gender":"M","address":"186 Oxford Walk","employer":"Datagen","email":"adkinsmejia@datagen.com","city":"Faywood","state":"OK","_is_real":true} +{"index":{"_id":"784"}} +{"account_number":784,"balance":25291,"firstname":"Mabel","lastname":"Thornton","age":21,"gender":"M","address":"124 Louisiana Avenue","employer":"Zolavo","email":"mabelthornton@zolavo.com","city":"Lynn","state":"AL","_is_real":true} +{"index":{"_id":"789"}} +{"account_number":789,"balance":8760,"firstname":"Cunningham","lastname":"Kerr","age":27,"gender":"F","address":"154 Sharon Street","employer":"Polarium","email":"cunninghamkerr@polarium.com","city":"Tuskahoma","state":"MS","_is_real":true} +{"index":{"_id":"791"}} +{"account_number":791,"balance":48249,"firstname":"Janine","lastname":"Huber","age":38,"gender":"F","address":"348 Porter Avenue","employer":"Viocular","email":"janinehuber@viocular.com","city":"Fivepointville","state":"MA","_is_real":true} +{"index":{"_id":"796"}} +{"account_number":796,"balance":23503,"firstname":"Mona","lastname":"Craft","age":35,"gender":"F","address":"511 Henry Street","employer":"Opticom","email":"monacraft@opticom.com","city":"Websterville","state":"IN","_is_real":true} +{"index":{"_id":"804"}} +{"account_number":804,"balance":23610,"firstname":"Rojas","lastname":"Oneal","age":27,"gender":"M","address":"669 Sandford Street","employer":"Glukgluk","email":"rojasoneal@glukgluk.com","city":"Wheaton","state":"ME","_is_real":true} +{"index":{"_id":"809"}} +{"account_number":809,"balance":47812,"firstname":"Christie","lastname":"Strickland","age":30,"gender":"M","address":"346 Bancroft Place","employer":"Anarco","email":"christiestrickland@anarco.com","city":"Baden","state":"NV","_is_real":true} +{"index":{"_id":"811"}} +{"account_number":811,"balance":26007,"firstname":"Walls","lastname":"Rogers","age":28,"gender":"F","address":"352 Freeman Street","employer":"Geekmosis","email":"wallsrogers@geekmosis.com","city":"Caroleen","state":"NV","_is_real":true} +{"index":{"_id":"816"}} +{"account_number":816,"balance":9567,"firstname":"Cornelia","lastname":"Lane","age":20,"gender":"F","address":"384 Bainbridge Street","employer":"Sulfax","email":"cornelialane@sulfax.com","city":"Elizaville","state":"MS","_is_real":true} +{"index":{"_id":"823"}} +{"account_number":823,"balance":48726,"firstname":"Celia","lastname":"Bernard","age":33,"gender":"F","address":"466 Amboy Street","employer":"Mitroc","email":"celiabernard@mitroc.com","city":"Skyland","state":"GA","_is_real":true} +{"index":{"_id":"828"}} +{"account_number":828,"balance":44890,"firstname":"Blanche","lastname":"Holmes","age":33,"gender":"F","address":"605 Stryker Court","employer":"Motovate","email":"blancheholmes@motovate.com","city":"Loomis","state":"KS","_is_real":true} +{"index":{"_id":"830"}} +{"account_number":830,"balance":45210,"firstname":"Louella","lastname":"Chan","age":23,"gender":"M","address":"511 Heath Place","employer":"Conferia","email":"louellachan@conferia.com","city":"Brookfield","state":"OK","_is_real":true} +{"index":{"_id":"835"}} +{"account_number":835,"balance":46558,"firstname":"Glover","lastname":"Rutledge","age":25,"gender":"F","address":"641 Royce Street","employer":"Ginkogene","email":"gloverrutledge@ginkogene.com","city":"Dixonville","state":"VA","_is_real":true} +{"index":{"_id":"842"}} +{"account_number":842,"balance":49587,"firstname":"Meagan","lastname":"Buckner","age":23,"gender":"F","address":"833 Bushwick Court","employer":"Biospan","email":"meaganbuckner@biospan.com","city":"Craig","state":"TX","_is_real":true} +{"index":{"_id":"847"}} +{"account_number":847,"balance":8652,"firstname":"Antonia","lastname":"Duncan","age":23,"gender":"M","address":"644 Stryker Street","employer":"Talae","email":"antoniaduncan@talae.com","city":"Dawn","state":"MO","_is_real":true} +{"index":{"_id":"854"}} +{"account_number":854,"balance":49795,"firstname":"Jimenez","lastname":"Barry","age":25,"gender":"F","address":"603 Cooper Street","employer":"Verton","email":"jimenezbarry@verton.com","city":"Moscow","state":"AL","_is_real":true} +{"index":{"_id":"859"}} +{"account_number":859,"balance":20734,"firstname":"Beulah","lastname":"Stuart","age":24,"gender":"F","address":"651 Albemarle Terrace","employer":"Hatology","email":"beulahstuart@hatology.com","city":"Waiohinu","state":"RI","_is_real":true} +{"index":{"_id":"861"}} +{"account_number":861,"balance":44173,"firstname":"Jaime","lastname":"Wilson","age":35,"gender":"M","address":"680 Richardson Street","employer":"Temorak","email":"jaimewilson@temorak.com","city":"Fidelis","state":"FL","_is_real":true} +{"index":{"_id":"866"}} +{"account_number":866,"balance":45565,"firstname":"Araceli","lastname":"Woodward","age":28,"gender":"M","address":"326 Meadow Street","employer":"Olympix","email":"araceliwoodward@olympix.com","city":"Dana","state":"KS","_is_real":true} +{"index":{"_id":"873"}} +{"account_number":873,"balance":43931,"firstname":"Tisha","lastname":"Cotton","age":39,"gender":"F","address":"432 Lincoln Road","employer":"Buzzmaker","email":"tishacotton@buzzmaker.com","city":"Bluetown","state":"GA","_is_real":true} +{"index":{"_id":"878"}} +{"account_number":878,"balance":49159,"firstname":"Battle","lastname":"Blackburn","age":40,"gender":"F","address":"234 Hendrix Street","employer":"Zilphur","email":"battleblackburn@zilphur.com","city":"Wanamie","state":"PA","_is_real":true} +{"index":{"_id":"880"}} +{"account_number":880,"balance":22575,"firstname":"Christian","lastname":"Myers","age":35,"gender":"M","address":"737 Crown Street","employer":"Combogen","email":"christianmyers@combogen.com","city":"Abrams","state":"OK","_is_real":true} +{"index":{"_id":"885"}} +{"account_number":885,"balance":31661,"firstname":"Valdez","lastname":"Roberson","age":40,"gender":"F","address":"227 Scholes Street","employer":"Delphide","email":"valdezroberson@delphide.com","city":"Chilton","state":"MT","_is_real":true} +{"index":{"_id":"892"}} +{"account_number":892,"balance":44974,"firstname":"Hill","lastname":"Hayes","age":29,"gender":"M","address":"721 Dooley Street","employer":"Fuelton","email":"hillhayes@fuelton.com","city":"Orason","state":"MT","_is_real":true} +{"index":{"_id":"897"}} +{"account_number":897,"balance":45973,"firstname":"Alyson","lastname":"Irwin","age":25,"gender":"M","address":"731 Poplar Street","employer":"Quizka","email":"alysonirwin@quizka.com","city":"Singer","state":"VA","_is_real":true} +{"index":{"_id":"900"}} +{"account_number":900,"balance":6124,"firstname":"Gonzalez","lastname":"Watson","age":23,"gender":"M","address":"624 Sullivan Street","employer":"Marvane","email":"gonzalezwatson@marvane.com","city":"Wikieup","state":"IL","_is_real":true} +{"index":{"_id":"905"}} +{"account_number":905,"balance":29438,"firstname":"Schultz","lastname":"Moreno","age":20,"gender":"F","address":"761 Cedar Street","employer":"Paragonia","email":"schultzmoreno@paragonia.com","city":"Glenshaw","state":"SC","_is_real":true} +{"index":{"_id":"912"}} +{"account_number":912,"balance":13675,"firstname":"Flora","lastname":"Alvarado","age":26,"gender":"M","address":"771 Vandervoort Avenue","employer":"Boilicon","email":"floraalvarado@boilicon.com","city":"Vivian","state":"ID","_is_real":true} +{"index":{"_id":"917"}} +{"account_number":917,"balance":47782,"firstname":"Parks","lastname":"Hurst","age":24,"gender":"M","address":"933 Cozine Avenue","employer":"Pyramis","email":"parkshurst@pyramis.com","city":"Lindcove","state":"GA","_is_real":true} +{"index":{"_id":"924"}} +{"account_number":924,"balance":3811,"firstname":"Hilary","lastname":"Leonard","age":24,"gender":"M","address":"235 Hegeman Avenue","employer":"Metroz","email":"hilaryleonard@metroz.com","city":"Roosevelt","state":"ME","_is_real":true} +{"index":{"_id":"929"}} +{"account_number":929,"balance":34708,"firstname":"Willie","lastname":"Hickman","age":35,"gender":"M","address":"430 Devoe Street","employer":"Apextri","email":"williehickman@apextri.com","city":"Clay","state":"MS","_is_real":true} +{"index":{"_id":"931"}} +{"account_number":931,"balance":8244,"firstname":"Ingrid","lastname":"Garcia","age":23,"gender":"F","address":"674 Indiana Place","employer":"Balooba","email":"ingridgarcia@balooba.com","city":"Interlochen","state":"AZ","_is_real":true} +{"index":{"_id":"936"}} +{"account_number":936,"balance":22430,"firstname":"Beth","lastname":"Frye","age":36,"gender":"M","address":"462 Thatford Avenue","employer":"Puria","email":"bethfrye@puria.com","city":"Hiseville","state":"LA","_is_real":true} +{"index":{"_id":"943"}} +{"account_number":943,"balance":24187,"firstname":"Wagner","lastname":"Griffin","age":23,"gender":"M","address":"489 Ellery Street","employer":"Gazak","email":"wagnergriffin@gazak.com","city":"Lorraine","state":"HI","_is_real":true} +{"index":{"_id":"948"}} +{"account_number":948,"balance":37074,"firstname":"Sargent","lastname":"Powers","age":40,"gender":"M","address":"532 Fiske Place","employer":"Accuprint","email":"sargentpowers@accuprint.com","city":"Umapine","state":"AK","_is_real":true} +{"index":{"_id":"950"}} +{"account_number":950,"balance":30916,"firstname":"Sherrie","lastname":"Patel","age":32,"gender":"F","address":"658 Langham Street","employer":"Futurize","email":"sherriepatel@futurize.com","city":"Garfield","state":"OR","_is_real":true} +{"index":{"_id":"955"}} +{"account_number":955,"balance":41621,"firstname":"Klein","lastname":"Kemp","age":33,"gender":"M","address":"370 Vanderbilt Avenue","employer":"Synkgen","email":"kleinkemp@synkgen.com","city":"Bonanza","state":"FL","_is_real":true} +{"index":{"_id":"962"}} +{"account_number":962,"balance":32096,"firstname":"Trujillo","lastname":"Wilcox","age":21,"gender":"F","address":"914 Duffield Street","employer":"Extragene","email":"trujillowilcox@extragene.com","city":"Golconda","state":"MA","_is_real":true} +{"index":{"_id":"967"}} +{"account_number":967,"balance":19161,"firstname":"Carrie","lastname":"Huffman","age":36,"gender":"F","address":"240 Sands Street","employer":"Injoy","email":"carriehuffman@injoy.com","city":"Leroy","state":"CA","_is_real":true} +{"index":{"_id":"974"}} +{"account_number":974,"balance":38082,"firstname":"Deborah","lastname":"Yang","age":26,"gender":"F","address":"463 Goodwin Place","employer":"Entogrok","email":"deborahyang@entogrok.com","city":"Herald","state":"KY","_is_real":true} +{"index":{"_id":"979"}} +{"account_number":979,"balance":43130,"firstname":"Vaughn","lastname":"Pittman","age":29,"gender":"M","address":"446 Tompkins Place","employer":"Phormula","email":"vaughnpittman@phormula.com","city":"Fingerville","state":"WI","_is_real":true} +{"index":{"_id":"981"}} +{"account_number":981,"balance":20278,"firstname":"Nolan","lastname":"Warner","age":29,"gender":"F","address":"753 Channel Avenue","employer":"Interodeo","email":"nolanwarner@interodeo.com","city":"Layhill","state":"MT","_is_real":true} +{"index":{"_id":"986"}} +{"account_number":986,"balance":35086,"firstname":"Norris","lastname":"Hubbard","age":31,"gender":"M","address":"600 Celeste Court","employer":"Printspan","email":"norrishubbard@printspan.com","city":"Cassel","state":"MI","_is_real":true} +{"index":{"_id":"993"}} +{"account_number":993,"balance":26487,"firstname":"Campos","lastname":"Olsen","age":37,"gender":"M","address":"873 Covert Street","employer":"Isbol","email":"camposolsen@isbol.com","city":"Glendale","state":"AK","_is_real":true} +{"index":{"_id":"998"}} +{"account_number":998,"balance":16869,"firstname":"Letha","lastname":"Baker","age":40,"gender":"F","address":"206 Llama Court","employer":"Dognosis","email":"lethabaker@dognosis.com","city":"Dunlo","state":"WV","_is_real":true} +{"index":{"_id":"2"}} +{"account_number":2,"balance":28838,"firstname":"Roberta","lastname":"Bender","age":22,"gender":"F","address":"560 Kingsway Place","employer":"Chillium","email":"robertabender@chillium.com","city":"Bennett","state":"LA","_is_real":true} +{"index":{"_id":"7"}} +{"account_number":7,"balance":39121,"firstname":"Levy","lastname":"Richard","age":22,"gender":"M","address":"820 Logan Street","employer":"Teraprene","email":"levyrichard@teraprene.com","city":"Shrewsbury","state":"MO","_is_real":true} +{"index":{"_id":"14"}} +{"account_number":14,"balance":20480,"firstname":"Erma","lastname":"Kane","age":39,"gender":"F","address":"661 Vista Place","employer":"Stockpost","email":"ermakane@stockpost.com","city":"Chamizal","state":"NY","_is_real":true} +{"index":{"_id":"19"}} +{"account_number":19,"balance":27894,"firstname":"Schwartz","lastname":"Buchanan","age":28,"gender":"F","address":"449 Mersereau Court","employer":"Sybixtex","email":"schwartzbuchanan@sybixtex.com","city":"Greenwich","state":"KS","_is_real":true} +{"index":{"_id":"21"}} +{"account_number":21,"balance":7004,"firstname":"Estella","lastname":"Paul","age":38,"gender":"M","address":"859 Portal Street","employer":"Zillatide","email":"estellapaul@zillatide.com","city":"Churchill","state":"WV","_is_real":true} +{"index":{"_id":"26"}} +{"account_number":26,"balance":14127,"firstname":"Lorraine","lastname":"Mccullough","age":39,"gender":"F","address":"157 Dupont Street","employer":"Zosis","email":"lorrainemccullough@zosis.com","city":"Dennard","state":"NH","_is_real":true} +{"index":{"_id":"33"}} +{"account_number":33,"balance":35439,"firstname":"Savannah","lastname":"Kirby","age":30,"gender":"F","address":"372 Malta Street","employer":"Musanpoly","email":"savannahkirby@musanpoly.com","city":"Muse","state":"AK","_is_real":true} +{"index":{"_id":"38"}} +{"account_number":38,"balance":10511,"firstname":"Erna","lastname":"Fields","age":32,"gender":"M","address":"357 Maple Street","employer":"Eweville","email":"ernafields@eweville.com","city":"Twilight","state":"MS","_is_real":true} +{"index":{"_id":"40"}} +{"account_number":40,"balance":33882,"firstname":"Pace","lastname":"Molina","age":40,"gender":"M","address":"263 Ovington Court","employer":"Cytrak","email":"pacemolina@cytrak.com","city":"Silkworth","state":"OR","_is_real":true} +{"index":{"_id":"45"}} +{"account_number":45,"balance":44478,"firstname":"Geneva","lastname":"Morin","age":21,"gender":"F","address":"357 Herkimer Street","employer":"Ezent","email":"genevamorin@ezent.com","city":"Blanco","state":"AZ","_is_real":true} +{"index":{"_id":"52"}} +{"account_number":52,"balance":46425,"firstname":"Kayla","lastname":"Bradshaw","age":31,"gender":"M","address":"449 Barlow Drive","employer":"Magnemo","email":"kaylabradshaw@magnemo.com","city":"Wawona","state":"AZ","_is_real":true} +{"index":{"_id":"57"}} +{"account_number":57,"balance":8705,"firstname":"Powell","lastname":"Herring","age":21,"gender":"M","address":"263 Merit Court","employer":"Digiprint","email":"powellherring@digiprint.com","city":"Coral","state":"MT","_is_real":true} +{"index":{"_id":"64"}} +{"account_number":64,"balance":44036,"firstname":"Miles","lastname":"Battle","age":35,"gender":"F","address":"988 Homecrest Avenue","employer":"Koffee","email":"milesbattle@koffee.com","city":"Motley","state":"ID","_is_real":true} +{"index":{"_id":"69"}} +{"account_number":69,"balance":14253,"firstname":"Desiree","lastname":"Harrison","age":24,"gender":"M","address":"694 Garland Court","employer":"Barkarama","email":"desireeharrison@barkarama.com","city":"Hackneyville","state":"GA","_is_real":true} +{"index":{"_id":"71"}} +{"account_number":71,"balance":38201,"firstname":"Sharpe","lastname":"Hoffman","age":39,"gender":"F","address":"450 Conklin Avenue","employer":"Centree","email":"sharpehoffman@centree.com","city":"Urbana","state":"WY","_is_real":true} +{"index":{"_id":"76"}} +{"account_number":76,"balance":38345,"firstname":"Claudette","lastname":"Beard","age":24,"gender":"F","address":"748 Dorset Street","employer":"Repetwire","email":"claudettebeard@repetwire.com","city":"Caln","state":"TX","_is_real":true} +{"index":{"_id":"83"}} +{"account_number":83,"balance":35928,"firstname":"Mayo","lastname":"Cleveland","age":28,"gender":"M","address":"720 Brooklyn Road","employer":"Indexia","email":"mayocleveland@indexia.com","city":"Roberts","state":"ND","_is_real":true} +{"index":{"_id":"88"}} +{"account_number":88,"balance":26418,"firstname":"Adela","lastname":"Tyler","age":21,"gender":"F","address":"737 Clove Road","employer":"Surelogic","email":"adelatyler@surelogic.com","city":"Boling","state":"SD","_is_real":true} +{"index":{"_id":"90"}} +{"account_number":90,"balance":25332,"firstname":"Herman","lastname":"Snyder","age":22,"gender":"F","address":"737 College Place","employer":"Lunchpod","email":"hermansnyder@lunchpod.com","city":"Flintville","state":"IA","_is_real":true} +{"index":{"_id":"95"}} +{"account_number":95,"balance":1650,"firstname":"Dominguez","lastname":"Le","age":20,"gender":"M","address":"539 Grace Court","employer":"Portica","email":"dominguezle@portica.com","city":"Wollochet","state":"KS","_is_real":true} +{"index":{"_id":"103"}} +{"account_number":103,"balance":11253,"firstname":"Calhoun","lastname":"Bruce","age":33,"gender":"F","address":"731 Clarkson Avenue","employer":"Automon","email":"calhounbruce@automon.com","city":"Marienthal","state":"IL","_is_real":true} +{"index":{"_id":"108"}} +{"account_number":108,"balance":19015,"firstname":"Christensen","lastname":"Weaver","age":21,"gender":"M","address":"398 Dearborn Court","employer":"Quilk","email":"christensenweaver@quilk.com","city":"Belvoir","state":"TX","_is_real":true} +{"index":{"_id":"110"}} +{"account_number":110,"balance":4850,"firstname":"Daphne","lastname":"Byrd","age":23,"gender":"F","address":"239 Conover Street","employer":"Freakin","email":"daphnebyrd@freakin.com","city":"Taft","state":"MN","_is_real":true} +{"index":{"_id":"115"}} +{"account_number":115,"balance":18750,"firstname":"Nikki","lastname":"Doyle","age":31,"gender":"F","address":"537 Clara Street","employer":"Fossiel","email":"nikkidoyle@fossiel.com","city":"Caron","state":"MS","_is_real":true} +{"index":{"_id":"122"}} +{"account_number":122,"balance":17128,"firstname":"Aurora","lastname":"Fry","age":31,"gender":"F","address":"227 Knapp Street","employer":"Makingway","email":"aurorafry@makingway.com","city":"Maybell","state":"NE","_is_real":true} +{"index":{"_id":"127"}} +{"account_number":127,"balance":48734,"firstname":"Diann","lastname":"Mclaughlin","age":33,"gender":"F","address":"340 Clermont Avenue","employer":"Enomen","email":"diannmclaughlin@enomen.com","city":"Rutherford","state":"ND","_is_real":true} +{"index":{"_id":"134"}} +{"account_number":134,"balance":33829,"firstname":"Madelyn","lastname":"Norris","age":30,"gender":"F","address":"176 Noel Avenue","employer":"Endicil","email":"madelynnorris@endicil.com","city":"Walker","state":"NE","_is_real":true} +{"index":{"_id":"139"}} +{"account_number":139,"balance":18444,"firstname":"Rios","lastname":"Todd","age":35,"gender":"F","address":"281 Georgia Avenue","employer":"Uberlux","email":"riostodd@uberlux.com","city":"Hannasville","state":"PA","_is_real":true} +{"index":{"_id":"141"}} +{"account_number":141,"balance":20790,"firstname":"Liliana","lastname":"Caldwell","age":29,"gender":"M","address":"414 Huron Street","employer":"Rubadub","email":"lilianacaldwell@rubadub.com","city":"Hiwasse","state":"OK","_is_real":true} +{"index":{"_id":"146"}} +{"account_number":146,"balance":39078,"firstname":"Lang","lastname":"Kaufman","age":32,"gender":"F","address":"626 Beverley Road","employer":"Rodeomad","email":"langkaufman@rodeomad.com","city":"Mahtowa","state":"RI","_is_real":true} +{"index":{"_id":"153"}} +{"account_number":153,"balance":32074,"firstname":"Bird","lastname":"Cochran","age":31,"gender":"F","address":"691 Bokee Court","employer":"Supremia","email":"birdcochran@supremia.com","city":"Barrelville","state":"NE","_is_real":true} +{"index":{"_id":"158"}} +{"account_number":158,"balance":9380,"firstname":"Natalie","lastname":"Mcdowell","age":27,"gender":"M","address":"953 Roder Avenue","employer":"Myopium","email":"nataliemcdowell@myopium.com","city":"Savage","state":"ND","_is_real":true} +{"index":{"_id":"160"}} +{"account_number":160,"balance":48974,"firstname":"Hull","lastname":"Cherry","age":23,"gender":"F","address":"275 Beaumont Street","employer":"Noralex","email":"hullcherry@noralex.com","city":"Whipholt","state":"WA","_is_real":true} +{"index":{"_id":"165"}} +{"account_number":165,"balance":18956,"firstname":"Sims","lastname":"Mckay","age":40,"gender":"F","address":"205 Jackson Street","employer":"Comtour","email":"simsmckay@comtour.com","city":"Tilden","state":"DC","_is_real":true} +{"index":{"_id":"172"}} +{"account_number":172,"balance":18356,"firstname":"Marie","lastname":"Whitehead","age":20,"gender":"M","address":"704 Monaco Place","employer":"Sultrax","email":"mariewhitehead@sultrax.com","city":"Dragoon","state":"IL","_is_real":true} +{"index":{"_id":"177"}} +{"account_number":177,"balance":48972,"firstname":"Harris","lastname":"Gross","age":40,"gender":"F","address":"468 Suydam Street","employer":"Kidstock","email":"harrisgross@kidstock.com","city":"Yettem","state":"KY","_is_real":true} +{"index":{"_id":"184"}} +{"account_number":184,"balance":9157,"firstname":"Cathy","lastname":"Morrison","age":27,"gender":"M","address":"882 Pine Street","employer":"Zytrek","email":"cathymorrison@zytrek.com","city":"Fedora","state":"FL","_is_real":true} +{"index":{"_id":"189"}} +{"account_number":189,"balance":20167,"firstname":"Ada","lastname":"Cortez","age":38,"gender":"F","address":"700 Forest Place","employer":"Micronaut","email":"adacortez@micronaut.com","city":"Eagletown","state":"TX","_is_real":true} +{"index":{"_id":"191"}} +{"account_number":191,"balance":26172,"firstname":"Barr","lastname":"Sharpe","age":28,"gender":"M","address":"428 Auburn Place","employer":"Ziggles","email":"barrsharpe@ziggles.com","city":"Springdale","state":"KS","_is_real":true} +{"index":{"_id":"196"}} +{"account_number":196,"balance":29931,"firstname":"Caldwell","lastname":"Daniel","age":28,"gender":"F","address":"405 Oliver Street","employer":"Furnigeer","email":"caldwelldaniel@furnigeer.com","city":"Zortman","state":"NE","_is_real":true} +{"index":{"_id":"204"}} +{"account_number":204,"balance":27714,"firstname":"Mavis","lastname":"Deleon","age":39,"gender":"F","address":"400 Waldane Court","employer":"Lotron","email":"mavisdeleon@lotron.com","city":"Stollings","state":"LA","_is_real":true} +{"index":{"_id":"209"}} +{"account_number":209,"balance":31052,"firstname":"Myers","lastname":"Noel","age":30,"gender":"F","address":"691 Alton Place","employer":"Greeker","email":"myersnoel@greeker.com","city":"Hinsdale","state":"KY","_is_real":true} +{"index":{"_id":"211"}} +{"account_number":211,"balance":21539,"firstname":"Graciela","lastname":"Vaughan","age":22,"gender":"M","address":"558 Montauk Court","employer":"Fishland","email":"gracielavaughan@fishland.com","city":"Madrid","state":"PA","_is_real":true} +{"index":{"_id":"216"}} +{"account_number":216,"balance":11422,"firstname":"Price","lastname":"Haley","age":35,"gender":"M","address":"233 Portland Avenue","employer":"Zeam","email":"pricehaley@zeam.com","city":"Titanic","state":"UT","_is_real":true} +{"index":{"_id":"223"}} +{"account_number":223,"balance":9528,"firstname":"Newton","lastname":"Fletcher","age":26,"gender":"F","address":"654 Dewitt Avenue","employer":"Assistia","email":"newtonfletcher@assistia.com","city":"Nipinnawasee","state":"AK","_is_real":true} +{"index":{"_id":"228"}} +{"account_number":228,"balance":10543,"firstname":"Rosella","lastname":"Albert","age":20,"gender":"M","address":"185 Gotham Avenue","employer":"Isoplex","email":"rosellaalbert@isoplex.com","city":"Finzel","state":"NY","_is_real":true} +{"index":{"_id":"230"}} +{"account_number":230,"balance":10829,"firstname":"Chris","lastname":"Raymond","age":28,"gender":"F","address":"464 Remsen Street","employer":"Cogentry","email":"chrisraymond@cogentry.com","city":"Bowmansville","state":"SD","_is_real":true} +{"index":{"_id":"235"}} +{"account_number":235,"balance":17729,"firstname":"Mcpherson","lastname":"Mueller","age":31,"gender":"M","address":"541 Strong Place","employer":"Tingles","email":"mcphersonmueller@tingles.com","city":"Brantleyville","state":"AR","_is_real":true} +{"index":{"_id":"242"}} +{"account_number":242,"balance":42318,"firstname":"Berger","lastname":"Roach","age":21,"gender":"M","address":"125 Wakeman Place","employer":"Ovium","email":"bergerroach@ovium.com","city":"Hessville","state":"WI","_is_real":true} +{"index":{"_id":"247"}} +{"account_number":247,"balance":45123,"firstname":"Mccormick","lastname":"Moon","age":37,"gender":"M","address":"582 Brighton Avenue","employer":"Norsup","email":"mccormickmoon@norsup.com","city":"Forestburg","state":"DE","_is_real":true} +{"index":{"_id":"254"}} +{"account_number":254,"balance":35104,"firstname":"Yang","lastname":"Dodson","age":21,"gender":"M","address":"531 Lott Street","employer":"Mondicil","email":"yangdodson@mondicil.com","city":"Enoree","state":"UT","_is_real":true} +{"index":{"_id":"259"}} +{"account_number":259,"balance":41877,"firstname":"Eleanor","lastname":"Gonzalez","age":30,"gender":"M","address":"800 Sumpter Street","employer":"Futuris","email":"eleanorgonzalez@futuris.com","city":"Jenkinsville","state":"ID","_is_real":true} +{"index":{"_id":"261"}} +{"account_number":261,"balance":39998,"firstname":"Millicent","lastname":"Pickett","age":34,"gender":"F","address":"722 Montieth Street","employer":"Gushkool","email":"millicentpickett@gushkool.com","city":"Norwood","state":"MS","_is_real":true} +{"index":{"_id":"266"}} +{"account_number":266,"balance":2777,"firstname":"Monique","lastname":"Conner","age":35,"gender":"F","address":"489 Metrotech Courtr","employer":"Flotonic","email":"moniqueconner@flotonic.com","city":"Retsof","state":"MD","_is_real":true} +{"index":{"_id":"273"}} +{"account_number":273,"balance":11181,"firstname":"Murphy","lastname":"Chandler","age":20,"gender":"F","address":"569 Bradford Street","employer":"Zilch","email":"murphychandler@zilch.com","city":"Vicksburg","state":"FL","_is_real":true} +{"index":{"_id":"278"}} +{"account_number":278,"balance":22530,"firstname":"Tamra","lastname":"Navarro","age":27,"gender":"F","address":"175 Woodruff Avenue","employer":"Norsul","email":"tamranavarro@norsul.com","city":"Glasgow","state":"VT","_is_real":true} +{"index":{"_id":"280"}} +{"account_number":280,"balance":3380,"firstname":"Vilma","lastname":"Shields","age":26,"gender":"F","address":"133 Berriman Street","employer":"Applidec","email":"vilmashields@applidec.com","city":"Adamstown","state":"ME","_is_real":true} +{"index":{"_id":"285"}} +{"account_number":285,"balance":47369,"firstname":"Hilda","lastname":"Phillips","age":28,"gender":"F","address":"618 Nixon Court","employer":"Comcur","email":"hildaphillips@comcur.com","city":"Siglerville","state":"NC","_is_real":true} +{"index":{"_id":"292"}} +{"account_number":292,"balance":26679,"firstname":"Morrow","lastname":"Greene","age":20,"gender":"F","address":"691 Nassau Street","employer":"Columella","email":"morrowgreene@columella.com","city":"Sanborn","state":"FL","_is_real":true} +{"index":{"_id":"297"}} +{"account_number":297,"balance":20508,"firstname":"Tucker","lastname":"Patrick","age":35,"gender":"F","address":"978 Whitwell Place","employer":"Valreda","email":"tuckerpatrick@valreda.com","city":"Deseret","state":"CO","_is_real":true} +{"index":{"_id":"300"}} +{"account_number":300,"balance":25654,"firstname":"Lane","lastname":"Tate","age":26,"gender":"F","address":"632 Kay Court","employer":"Genesynk","email":"lanetate@genesynk.com","city":"Lowell","state":"MO","_is_real":true} +{"index":{"_id":"305"}} +{"account_number":305,"balance":11655,"firstname":"Augusta","lastname":"Winters","age":29,"gender":"F","address":"377 Paerdegat Avenue","employer":"Vendblend","email":"augustawinters@vendblend.com","city":"Gwynn","state":"MA","_is_real":true} +{"index":{"_id":"312"}} +{"account_number":312,"balance":8511,"firstname":"Burgess","lastname":"Gentry","age":25,"gender":"F","address":"382 Bergen Court","employer":"Orbixtar","email":"burgessgentry@orbixtar.com","city":"Conestoga","state":"WI","_is_real":true} +{"index":{"_id":"317"}} +{"account_number":317,"balance":31968,"firstname":"Ruiz","lastname":"Morris","age":31,"gender":"F","address":"972 Dean Street","employer":"Apex","email":"ruizmorris@apex.com","city":"Jacksonwald","state":"WV","_is_real":true} +{"index":{"_id":"324"}} +{"account_number":324,"balance":44976,"firstname":"Gladys","lastname":"Erickson","age":22,"gender":"M","address":"250 Battery Avenue","employer":"Eternis","email":"gladyserickson@eternis.com","city":"Marne","state":"IA","_is_real":true} +{"index":{"_id":"329"}} +{"account_number":329,"balance":31138,"firstname":"Nellie","lastname":"Mercer","age":25,"gender":"M","address":"967 Ebony Court","employer":"Scenty","email":"nelliemercer@scenty.com","city":"Jardine","state":"AK","_is_real":true} +{"index":{"_id":"331"}} +{"account_number":331,"balance":46004,"firstname":"Gibson","lastname":"Potts","age":34,"gender":"F","address":"994 Dahill Road","employer":"Zensus","email":"gibsonpotts@zensus.com","city":"Frizzleburg","state":"CO","_is_real":true} +{"index":{"_id":"336"}} +{"account_number":336,"balance":40891,"firstname":"Dudley","lastname":"Avery","age":25,"gender":"M","address":"405 Powers Street","employer":"Genmom","email":"dudleyavery@genmom.com","city":"Clarksburg","state":"CO","_is_real":true} +{"index":{"_id":"343"}} +{"account_number":343,"balance":37684,"firstname":"Robbie","lastname":"Logan","age":29,"gender":"M","address":"488 Linden Boulevard","employer":"Hydrocom","email":"robbielogan@hydrocom.com","city":"Stockdale","state":"TN","_is_real":true} +{"index":{"_id":"348"}} +{"account_number":348,"balance":1360,"firstname":"Karina","lastname":"Russell","age":37,"gender":"M","address":"797 Moffat Street","employer":"Limozen","email":"karinarussell@limozen.com","city":"Riegelwood","state":"RI","_is_real":true} +{"index":{"_id":"350"}} +{"account_number":350,"balance":4267,"firstname":"Wyatt","lastname":"Wise","age":22,"gender":"F","address":"896 Bleecker Street","employer":"Rockyard","email":"wyattwise@rockyard.com","city":"Joes","state":"MS","_is_real":true} +{"index":{"_id":"355"}} +{"account_number":355,"balance":40961,"firstname":"Gregory","lastname":"Delacruz","age":38,"gender":"M","address":"876 Cortelyou Road","employer":"Oulu","email":"gregorydelacruz@oulu.com","city":"Waterloo","state":"WV","_is_real":true} +{"index":{"_id":"362"}} +{"account_number":362,"balance":14938,"firstname":"Jimmie","lastname":"Dejesus","age":26,"gender":"M","address":"351 Navy Walk","employer":"Ecolight","email":"jimmiedejesus@ecolight.com","city":"Berlin","state":"ME","_is_real":true} +{"index":{"_id":"367"}} +{"account_number":367,"balance":40458,"firstname":"Elaine","lastname":"Workman","age":20,"gender":"M","address":"188 Ridge Boulevard","employer":"Colaire","email":"elaineworkman@colaire.com","city":"Herbster","state":"AK","_is_real":true} +{"index":{"_id":"374"}} +{"account_number":374,"balance":19521,"firstname":"Blanchard","lastname":"Stein","age":30,"gender":"M","address":"313 Bartlett Street","employer":"Cujo","email":"blanchardstein@cujo.com","city":"Cascades","state":"OR","_is_real":true} +{"index":{"_id":"379"}} +{"account_number":379,"balance":12962,"firstname":"Ruthie","lastname":"Lamb","age":21,"gender":"M","address":"796 Rockaway Avenue","employer":"Incubus","email":"ruthielamb@incubus.com","city":"Hickory","state":"TX","_is_real":true} +{"index":{"_id":"381"}} +{"account_number":381,"balance":40978,"firstname":"Sophie","lastname":"Mays","age":31,"gender":"M","address":"261 Varanda Place","employer":"Uneeq","email":"sophiemays@uneeq.com","city":"Cressey","state":"AR","_is_real":true} +{"index":{"_id":"386"}} +{"account_number":386,"balance":42588,"firstname":"Wallace","lastname":"Barr","age":39,"gender":"F","address":"246 Beverly Road","employer":"Concility","email":"wallacebarr@concility.com","city":"Durham","state":"IN","_is_real":true} +{"index":{"_id":"393"}} +{"account_number":393,"balance":43936,"firstname":"William","lastname":"Kelly","age":24,"gender":"M","address":"178 Lawrence Avenue","employer":"Techtrix","email":"williamkelly@techtrix.com","city":"Orin","state":"PA","_is_real":true} +{"index":{"_id":"398"}} +{"account_number":398,"balance":8543,"firstname":"Leticia","lastname":"Duran","age":35,"gender":"F","address":"305 Senator Street","employer":"Xleen","email":"leticiaduran@xleen.com","city":"Cavalero","state":"PA","_is_real":true} +{"index":{"_id":"401"}} +{"account_number":401,"balance":29408,"firstname":"Contreras","lastname":"Randolph","age":38,"gender":"M","address":"104 Lewis Avenue","employer":"Inrt","email":"contrerasrandolph@inrt.com","city":"Chesapeake","state":"CT","_is_real":true} +{"index":{"_id":"406"}} +{"account_number":406,"balance":28127,"firstname":"Mccarthy","lastname":"Dunlap","age":28,"gender":"F","address":"684 Seacoast Terrace","employer":"Canopoly","email":"mccarthydunlap@canopoly.com","city":"Elliott","state":"NC","_is_real":true} +{"index":{"_id":"413"}} +{"account_number":413,"balance":15631,"firstname":"Pugh","lastname":"Hamilton","age":39,"gender":"F","address":"124 Euclid Avenue","employer":"Techade","email":"pughhamilton@techade.com","city":"Beaulieu","state":"CA","_is_real":true} +{"index":{"_id":"418"}} +{"account_number":418,"balance":10207,"firstname":"Reed","lastname":"Goff","age":32,"gender":"M","address":"959 Everit Street","employer":"Zillan","email":"reedgoff@zillan.com","city":"Hiko","state":"WV","_is_real":true} +{"index":{"_id":"420"}} +{"account_number":420,"balance":44699,"firstname":"Brandie","lastname":"Hayden","age":22,"gender":"M","address":"291 Ash Street","employer":"Digifad","email":"brandiehayden@digifad.com","city":"Spelter","state":"NM","_is_real":true} +{"index":{"_id":"425"}} +{"account_number":425,"balance":41308,"firstname":"Queen","lastname":"Leach","age":30,"gender":"M","address":"105 Fair Street","employer":"Magneato","email":"queenleach@magneato.com","city":"Barronett","state":"NH","_is_real":true} +{"index":{"_id":"432"}} +{"account_number":432,"balance":28969,"firstname":"Preston","lastname":"Ferguson","age":40,"gender":"F","address":"239 Greenwood Avenue","employer":"Bitendrex","email":"prestonferguson@bitendrex.com","city":"Idledale","state":"ND","_is_real":true} +{"index":{"_id":"437"}} +{"account_number":437,"balance":41225,"firstname":"Rosales","lastname":"Marquez","age":29,"gender":"M","address":"873 Ryerson Street","employer":"Ronelon","email":"rosalesmarquez@ronelon.com","city":"Allendale","state":"CA","_is_real":true} +{"index":{"_id":"444"}} +{"account_number":444,"balance":44219,"firstname":"Dolly","lastname":"Finch","age":24,"gender":"F","address":"974 Interborough Parkway","employer":"Zytrac","email":"dollyfinch@zytrac.com","city":"Vowinckel","state":"WY","_is_real":true} +{"index":{"_id":"449"}} +{"account_number":449,"balance":41950,"firstname":"Barnett","lastname":"Cantrell","age":39,"gender":"F","address":"945 Bedell Lane","employer":"Zentility","email":"barnettcantrell@zentility.com","city":"Swartzville","state":"ND","_is_real":true} +{"index":{"_id":"451"}} +{"account_number":451,"balance":31950,"firstname":"Mason","lastname":"Mcleod","age":31,"gender":"F","address":"438 Havemeyer Street","employer":"Omatom","email":"masonmcleod@omatom.com","city":"Ryderwood","state":"NE","_is_real":true} +{"index":{"_id":"456"}} +{"account_number":456,"balance":21419,"firstname":"Solis","lastname":"Kline","age":33,"gender":"M","address":"818 Ashford Street","employer":"Vetron","email":"soliskline@vetron.com","city":"Ruffin","state":"NY","_is_real":true} +{"index":{"_id":"463"}} +{"account_number":463,"balance":36672,"firstname":"Heidi","lastname":"Acosta","age":20,"gender":"F","address":"692 Kenmore Terrace","employer":"Elpro","email":"heidiacosta@elpro.com","city":"Ezel","state":"SD","_is_real":true} +{"index":{"_id":"468"}} +{"account_number":468,"balance":18400,"firstname":"Foreman","lastname":"Fowler","age":40,"gender":"M","address":"443 Jackson Court","employer":"Zillactic","email":"foremanfowler@zillactic.com","city":"Wakarusa","state":"WA","_is_real":true} +{"index":{"_id":"470"}} +{"account_number":470,"balance":20455,"firstname":"Schneider","lastname":"Hull","age":35,"gender":"M","address":"724 Apollo Street","employer":"Exospeed","email":"schneiderhull@exospeed.com","city":"Watchtower","state":"ID","_is_real":true} +{"index":{"_id":"475"}} +{"account_number":475,"balance":24427,"firstname":"Morales","lastname":"Jacobs","age":22,"gender":"F","address":"225 Desmond Court","employer":"Oronoko","email":"moralesjacobs@oronoko.com","city":"Clayville","state":"CT","_is_real":true} +{"index":{"_id":"482"}} +{"account_number":482,"balance":14834,"firstname":"Janie","lastname":"Bass","age":39,"gender":"M","address":"781 Grattan Street","employer":"Manglo","email":"janiebass@manglo.com","city":"Kenwood","state":"IA","_is_real":true} +{"index":{"_id":"487"}} +{"account_number":487,"balance":30718,"firstname":"Sawyer","lastname":"Vincent","age":26,"gender":"F","address":"238 Lancaster Avenue","employer":"Brainquil","email":"sawyervincent@brainquil.com","city":"Galesville","state":"MS","_is_real":true} +{"index":{"_id":"494"}} +{"account_number":494,"balance":3592,"firstname":"Holden","lastname":"Bowen","age":30,"gender":"M","address":"374 Elmwood Avenue","employer":"Endipine","email":"holdenbowen@endipine.com","city":"Rosine","state":"ID","_is_real":true} +{"index":{"_id":"499"}} +{"account_number":499,"balance":26060,"firstname":"Lara","lastname":"Perkins","age":26,"gender":"M","address":"703 Monroe Street","employer":"Paprikut","email":"laraperkins@paprikut.com","city":"Barstow","state":"NY","_is_real":true} +{"index":{"_id":"502"}} +{"account_number":502,"balance":31898,"firstname":"Woodard","lastname":"Bailey","age":31,"gender":"F","address":"585 Albee Square","employer":"Imperium","email":"woodardbailey@imperium.com","city":"Matheny","state":"MT","_is_real":true} +{"index":{"_id":"507"}} +{"account_number":507,"balance":27675,"firstname":"Blankenship","lastname":"Ramirez","age":31,"gender":"M","address":"630 Graham Avenue","employer":"Bytrex","email":"blankenshipramirez@bytrex.com","city":"Bancroft","state":"CT","_is_real":true} +{"index":{"_id":"514"}} +{"account_number":514,"balance":30125,"firstname":"Solomon","lastname":"Bush","age":34,"gender":"M","address":"409 Harkness Avenue","employer":"Snacktion","email":"solomonbush@snacktion.com","city":"Grayhawk","state":"TX","_is_real":true} +{"index":{"_id":"519"}} +{"account_number":519,"balance":3282,"firstname":"Lorna","lastname":"Franco","age":31,"gender":"F","address":"722 Schenck Court","employer":"Zentia","email":"lornafranco@zentia.com","city":"National","state":"FL","_is_real":true} +{"index":{"_id":"521"}} +{"account_number":521,"balance":16348,"firstname":"Josefa","lastname":"Buckley","age":34,"gender":"F","address":"848 Taylor Street","employer":"Mazuda","email":"josefabuckley@mazuda.com","city":"Saranap","state":"NM","_is_real":true} +{"index":{"_id":"526"}} +{"account_number":526,"balance":35375,"firstname":"Sweeney","lastname":"Fulton","age":33,"gender":"F","address":"550 Martense Street","employer":"Cormoran","email":"sweeneyfulton@cormoran.com","city":"Chalfant","state":"IA","_is_real":true} +{"index":{"_id":"533"}} +{"account_number":533,"balance":13761,"firstname":"Margarita","lastname":"Diaz","age":23,"gender":"M","address":"295 Tapscott Street","employer":"Zilodyne","email":"margaritadiaz@zilodyne.com","city":"Hondah","state":"ID","_is_real":true} +{"index":{"_id":"538"}} +{"account_number":538,"balance":16416,"firstname":"Koch","lastname":"Barker","age":21,"gender":"M","address":"919 Gerry Street","employer":"Xplor","email":"kochbarker@xplor.com","city":"Dixie","state":"WY","_is_real":true} +{"index":{"_id":"540"}} +{"account_number":540,"balance":40235,"firstname":"Tammy","lastname":"Wiggins","age":32,"gender":"F","address":"186 Schenectady Avenue","employer":"Speedbolt","email":"tammywiggins@speedbolt.com","city":"Salvo","state":"LA","_is_real":true} +{"index":{"_id":"545"}} +{"account_number":545,"balance":27011,"firstname":"Lena","lastname":"Lucas","age":20,"gender":"M","address":"110 Lamont Court","employer":"Kindaloo","email":"lenalucas@kindaloo.com","city":"Harleigh","state":"KY","_is_real":true} +{"index":{"_id":"552"}} +{"account_number":552,"balance":14727,"firstname":"Kate","lastname":"Estes","age":39,"gender":"M","address":"785 Willmohr Street","employer":"Rodeocean","email":"kateestes@rodeocean.com","city":"Elfrida","state":"HI","_is_real":true} +{"index":{"_id":"557"}} +{"account_number":557,"balance":3119,"firstname":"Landry","lastname":"Buck","age":20,"gender":"M","address":"558 Schweikerts Walk","employer":"Protodyne","email":"landrybuck@protodyne.com","city":"Edneyville","state":"AL","_is_real":true} +{"index":{"_id":"564"}} +{"account_number":564,"balance":43631,"firstname":"Owens","lastname":"Bowers","age":22,"gender":"M","address":"842 Congress Street","employer":"Nspire","email":"owensbowers@nspire.com","city":"Machias","state":"VA","_is_real":true} +{"index":{"_id":"569"}} +{"account_number":569,"balance":40019,"firstname":"Sherri","lastname":"Rowe","age":39,"gender":"F","address":"591 Arlington Place","employer":"Netility","email":"sherrirowe@netility.com","city":"Bridgetown","state":"SC","_is_real":true} +{"index":{"_id":"571"}} +{"account_number":571,"balance":3014,"firstname":"Ayers","lastname":"Duffy","age":28,"gender":"F","address":"721 Wortman Avenue","employer":"Aquasseur","email":"ayersduffy@aquasseur.com","city":"Tilleda","state":"MS","_is_real":true} +{"index":{"_id":"576"}} +{"account_number":576,"balance":29682,"firstname":"Helena","lastname":"Robertson","age":33,"gender":"F","address":"774 Devon Avenue","employer":"Vicon","email":"helenarobertson@vicon.com","city":"Dyckesville","state":"NV","_is_real":true} +{"index":{"_id":"583"}} +{"account_number":583,"balance":26558,"firstname":"Castro","lastname":"West","age":34,"gender":"F","address":"814 Williams Avenue","employer":"Cipromox","email":"castrowest@cipromox.com","city":"Nescatunga","state":"IL","_is_real":true} +{"index":{"_id":"588"}} +{"account_number":588,"balance":43531,"firstname":"Martina","lastname":"Collins","age":31,"gender":"M","address":"301 Anna Court","employer":"Geekwagon","email":"martinacollins@geekwagon.com","city":"Oneida","state":"VA","_is_real":true} +{"index":{"_id":"590"}} +{"account_number":590,"balance":4652,"firstname":"Ladonna","lastname":"Tucker","age":31,"gender":"F","address":"162 Kane Place","employer":"Infotrips","email":"ladonnatucker@infotrips.com","city":"Utting","state":"IA","_is_real":true} +{"index":{"_id":"595"}} +{"account_number":595,"balance":12478,"firstname":"Mccall","lastname":"Britt","age":36,"gender":"F","address":"823 Hill Street","employer":"Cablam","email":"mccallbritt@cablam.com","city":"Vernon","state":"CA","_is_real":true} +{"index":{"_id":"603"}} +{"account_number":603,"balance":28145,"firstname":"Janette","lastname":"Guzman","age":31,"gender":"F","address":"976 Kingston Avenue","employer":"Splinx","email":"janetteguzman@splinx.com","city":"Boomer","state":"NC","_is_real":true} +{"index":{"_id":"608"}} +{"account_number":608,"balance":47091,"firstname":"Carey","lastname":"Whitley","age":32,"gender":"F","address":"976 Lawrence Street","employer":"Poshome","email":"careywhitley@poshome.com","city":"Weogufka","state":"NE","_is_real":true} +{"index":{"_id":"610"}} +{"account_number":610,"balance":40571,"firstname":"Foster","lastname":"Weber","age":24,"gender":"F","address":"323 Rochester Avenue","employer":"Firewax","email":"fosterweber@firewax.com","city":"Winston","state":"NY","_is_real":true} +{"index":{"_id":"615"}} +{"account_number":615,"balance":28726,"firstname":"Delgado","lastname":"Curry","age":28,"gender":"F","address":"706 Butler Street","employer":"Zoxy","email":"delgadocurry@zoxy.com","city":"Gracey","state":"SD","_is_real":true} +{"index":{"_id":"622"}} +{"account_number":622,"balance":9661,"firstname":"Paulette","lastname":"Hartman","age":38,"gender":"M","address":"375 Emerald Street","employer":"Locazone","email":"paulettehartman@locazone.com","city":"Canterwood","state":"OH","_is_real":true} +{"index":{"_id":"627"}} +{"account_number":627,"balance":47546,"firstname":"Crawford","lastname":"Sears","age":37,"gender":"F","address":"686 Eastern Parkway","employer":"Updat","email":"crawfordsears@updat.com","city":"Bison","state":"VT","_is_real":true} +{"index":{"_id":"634"}} +{"account_number":634,"balance":29805,"firstname":"Deloris","lastname":"Levy","age":38,"gender":"M","address":"838 Foster Avenue","employer":"Homelux","email":"delorislevy@homelux.com","city":"Kempton","state":"PA","_is_real":true} +{"index":{"_id":"639"}} +{"account_number":639,"balance":28875,"firstname":"Caitlin","lastname":"Clements","age":32,"gender":"F","address":"627 Aster Court","employer":"Bunga","email":"caitlinclements@bunga.com","city":"Cetronia","state":"SC","_is_real":true} +{"index":{"_id":"641"}} +{"account_number":641,"balance":18345,"firstname":"Sheppard","lastname":"Everett","age":39,"gender":"F","address":"791 Norwood Avenue","employer":"Roboid","email":"sheppardeverett@roboid.com","city":"Selma","state":"AK","_is_real":true} +{"index":{"_id":"646"}} +{"account_number":646,"balance":15559,"firstname":"Lavonne","lastname":"Reyes","age":31,"gender":"F","address":"983 Newport Street","employer":"Parcoe","email":"lavonnereyes@parcoe.com","city":"Monument","state":"LA","_is_real":true} +{"index":{"_id":"653"}} +{"account_number":653,"balance":7606,"firstname":"Marcia","lastname":"Bennett","age":33,"gender":"F","address":"455 Bragg Street","employer":"Opticall","email":"marciabennett@opticall.com","city":"Magnolia","state":"NC","_is_real":true} +{"index":{"_id":"658"}} +{"account_number":658,"balance":10210,"firstname":"Bass","lastname":"Mcconnell","age":32,"gender":"F","address":"274 Ocean Avenue","employer":"Combot","email":"bassmcconnell@combot.com","city":"Beyerville","state":"OH","_is_real":true} +{"index":{"_id":"660"}} +{"account_number":660,"balance":46427,"firstname":"Moon","lastname":"Wood","age":33,"gender":"F","address":"916 Amersfort Place","employer":"Olucore","email":"moonwood@olucore.com","city":"Como","state":"VA","_is_real":true} +{"index":{"_id":"665"}} +{"account_number":665,"balance":15215,"firstname":"Britney","lastname":"Young","age":36,"gender":"M","address":"766 Sackman Street","employer":"Geoforma","email":"britneyyoung@geoforma.com","city":"Tuttle","state":"WI","_is_real":true} +{"index":{"_id":"672"}} +{"account_number":672,"balance":12621,"firstname":"Camille","lastname":"Munoz","age":36,"gender":"F","address":"959 Lewis Place","employer":"Vantage","email":"camillemunoz@vantage.com","city":"Whitmer","state":"IN","_is_real":true} +{"index":{"_id":"677"}} +{"account_number":677,"balance":8491,"firstname":"Snider","lastname":"Benton","age":26,"gender":"M","address":"827 Evans Street","employer":"Medicroix","email":"sniderbenton@medicroix.com","city":"Kaka","state":"UT","_is_real":true} +{"index":{"_id":"684"}} +{"account_number":684,"balance":46091,"firstname":"Warren","lastname":"Snow","age":25,"gender":"M","address":"756 Oakland Place","employer":"Bizmatic","email":"warrensnow@bizmatic.com","city":"Hatteras","state":"NE","_is_real":true} +{"index":{"_id":"689"}} +{"account_number":689,"balance":14985,"firstname":"Ines","lastname":"Chaney","age":28,"gender":"M","address":"137 Dikeman Street","employer":"Zidant","email":"ineschaney@zidant.com","city":"Nettie","state":"DC","_is_real":true} +{"index":{"_id":"691"}} +{"account_number":691,"balance":10792,"firstname":"Mclean","lastname":"Colon","age":22,"gender":"M","address":"876 Classon Avenue","employer":"Elentrix","email":"mcleancolon@elentrix.com","city":"Unionville","state":"OK","_is_real":true} +{"index":{"_id":"696"}} +{"account_number":696,"balance":17568,"firstname":"Crane","lastname":"Matthews","age":32,"gender":"F","address":"721 Gerritsen Avenue","employer":"Intradisk","email":"cranematthews@intradisk.com","city":"Brewster","state":"WV","_is_real":true} +{"index":{"_id":"704"}} +{"account_number":704,"balance":45347,"firstname":"Peters","lastname":"Kent","age":22,"gender":"F","address":"871 Independence Avenue","employer":"Extragen","email":"peterskent@extragen.com","city":"Morriston","state":"CA","_is_real":true} +{"index":{"_id":"709"}} +{"account_number":709,"balance":11015,"firstname":"Abbott","lastname":"Odom","age":29,"gender":"M","address":"893 Union Street","employer":"Jimbies","email":"abbottodom@jimbies.com","city":"Leeper","state":"NJ","_is_real":true} +{"index":{"_id":"711"}} +{"account_number":711,"balance":26939,"firstname":"Villarreal","lastname":"Horton","age":35,"gender":"F","address":"861 Creamer Street","employer":"Lexicondo","email":"villarrealhorton@lexicondo.com","city":"Lydia","state":"MS","_is_real":true} +{"index":{"_id":"716"}} +{"account_number":716,"balance":19789,"firstname":"Paul","lastname":"Mason","age":34,"gender":"F","address":"618 Nichols Avenue","employer":"Slax","email":"paulmason@slax.com","city":"Snowville","state":"OK","_is_real":true} +{"index":{"_id":"723"}} +{"account_number":723,"balance":16421,"firstname":"Nixon","lastname":"Moran","age":27,"gender":"M","address":"569 Campus Place","employer":"Cuizine","email":"nixonmoran@cuizine.com","city":"Buxton","state":"DC","_is_real":true} +{"index":{"_id":"728"}} +{"account_number":728,"balance":44818,"firstname":"Conley","lastname":"Preston","age":28,"gender":"M","address":"450 Coventry Road","employer":"Obones","email":"conleypreston@obones.com","city":"Alden","state":"CO","_is_real":true} +{"index":{"_id":"730"}} +{"account_number":730,"balance":41299,"firstname":"Moore","lastname":"Lee","age":30,"gender":"M","address":"797 Turner Place","employer":"Orbean","email":"moorelee@orbean.com","city":"Highland","state":"DE","_is_real":true} +{"index":{"_id":"735"}} +{"account_number":735,"balance":3984,"firstname":"Loraine","lastname":"Willis","age":32,"gender":"F","address":"928 Grove Street","employer":"Gadtron","email":"lorainewillis@gadtron.com","city":"Lowgap","state":"NY","_is_real":true} +{"index":{"_id":"742"}} +{"account_number":742,"balance":24765,"firstname":"Merle","lastname":"Wooten","age":26,"gender":"M","address":"317 Pooles Lane","employer":"Tropolis","email":"merlewooten@tropolis.com","city":"Bentley","state":"ND","_is_real":true} +{"index":{"_id":"747"}} +{"account_number":747,"balance":16617,"firstname":"Diaz","lastname":"Austin","age":38,"gender":"M","address":"676 Harway Avenue","employer":"Irack","email":"diazaustin@irack.com","city":"Cliff","state":"HI","_is_real":true} +{"index":{"_id":"754"}} +{"account_number":754,"balance":10779,"firstname":"Jones","lastname":"Vega","age":25,"gender":"F","address":"795 India Street","employer":"Gluid","email":"jonesvega@gluid.com","city":"Tyhee","state":"FL","_is_real":true} +{"index":{"_id":"759"}} +{"account_number":759,"balance":38007,"firstname":"Rose","lastname":"Carlson","age":27,"gender":"M","address":"987 Navy Street","employer":"Aquasure","email":"rosecarlson@aquasure.com","city":"Carlton","state":"CT","_is_real":true} +{"index":{"_id":"761"}} +{"account_number":761,"balance":7663,"firstname":"Rae","lastname":"Juarez","age":34,"gender":"F","address":"560 Gilmore Court","employer":"Entropix","email":"raejuarez@entropix.com","city":"Northchase","state":"ID","_is_real":true} +{"index":{"_id":"766"}} +{"account_number":766,"balance":21957,"firstname":"Thomas","lastname":"Gillespie","age":38,"gender":"M","address":"993 Williams Place","employer":"Octocore","email":"thomasgillespie@octocore.com","city":"Defiance","state":"MS","_is_real":true} +{"index":{"_id":"773"}} +{"account_number":773,"balance":31126,"firstname":"Liza","lastname":"Coffey","age":36,"gender":"F","address":"540 Bulwer Place","employer":"Assurity","email":"lizacoffey@assurity.com","city":"Gilgo","state":"WV","_is_real":true} +{"index":{"_id":"778"}} +{"account_number":778,"balance":46007,"firstname":"Underwood","lastname":"Wheeler","age":28,"gender":"M","address":"477 Provost Street","employer":"Decratex","email":"underwoodwheeler@decratex.com","city":"Sardis","state":"ID","_is_real":true} +{"index":{"_id":"780"}} +{"account_number":780,"balance":4682,"firstname":"Maryanne","lastname":"Hendricks","age":26,"gender":"F","address":"709 Wolcott Street","employer":"Sarasonic","email":"maryannehendricks@sarasonic.com","city":"Santel","state":"NH","_is_real":true} +{"index":{"_id":"785"}} +{"account_number":785,"balance":25078,"firstname":"Fields","lastname":"Lester","age":29,"gender":"M","address":"808 Chestnut Avenue","employer":"Visualix","email":"fieldslester@visualix.com","city":"Rowe","state":"PA","_is_real":true} +{"index":{"_id":"792"}} +{"account_number":792,"balance":13109,"firstname":"Becky","lastname":"Jimenez","age":40,"gender":"F","address":"539 Front Street","employer":"Isologia","email":"beckyjimenez@isologia.com","city":"Summertown","state":"MI","_is_real":true} +{"index":{"_id":"797"}} +{"account_number":797,"balance":6854,"firstname":"Lindsay","lastname":"Mills","age":26,"gender":"F","address":"919 Quay Street","employer":"Zoinage","email":"lindsaymills@zoinage.com","city":"Elliston","state":"VA","_is_real":true} +{"index":{"_id":"800"}} +{"account_number":800,"balance":26217,"firstname":"Candy","lastname":"Oconnor","age":28,"gender":"M","address":"200 Newel Street","employer":"Radiantix","email":"candyoconnor@radiantix.com","city":"Sandston","state":"OH","_is_real":true} +{"index":{"_id":"805"}} +{"account_number":805,"balance":18426,"firstname":"Jackson","lastname":"Sampson","age":27,"gender":"F","address":"722 Kenmore Court","employer":"Daido","email":"jacksonsampson@daido.com","city":"Bellamy","state":"ME","_is_real":true} +{"index":{"_id":"812"}} +{"account_number":812,"balance":42593,"firstname":"Graves","lastname":"Newman","age":32,"gender":"F","address":"916 Joralemon Street","employer":"Ecrater","email":"gravesnewman@ecrater.com","city":"Crown","state":"PA","_is_real":true} +{"index":{"_id":"817"}} +{"account_number":817,"balance":36582,"firstname":"Padilla","lastname":"Bauer","age":36,"gender":"F","address":"310 Cadman Plaza","employer":"Exoblue","email":"padillabauer@exoblue.com","city":"Ahwahnee","state":"MN","_is_real":true} +{"index":{"_id":"824"}} +{"account_number":824,"balance":6053,"firstname":"Dyer","lastname":"Henson","age":33,"gender":"M","address":"650 Seaview Avenue","employer":"Nitracyr","email":"dyerhenson@nitracyr.com","city":"Gibsonia","state":"KS","_is_real":true} +{"index":{"_id":"829"}} +{"account_number":829,"balance":20263,"firstname":"Althea","lastname":"Bell","age":37,"gender":"M","address":"319 Cook Street","employer":"Hyplex","email":"altheabell@hyplex.com","city":"Wadsworth","state":"DC","_is_real":true} +{"index":{"_id":"831"}} +{"account_number":831,"balance":25375,"firstname":"Wendy","lastname":"Savage","age":37,"gender":"M","address":"421 Veranda Place","employer":"Neurocell","email":"wendysavage@neurocell.com","city":"Fresno","state":"MS","_is_real":true} +{"index":{"_id":"836"}} +{"account_number":836,"balance":20797,"firstname":"Lloyd","lastname":"Lindsay","age":25,"gender":"F","address":"953 Dinsmore Place","employer":"Suretech","email":"lloydlindsay@suretech.com","city":"Conway","state":"VA","_is_real":true} +{"index":{"_id":"843"}} +{"account_number":843,"balance":15555,"firstname":"Patricia","lastname":"Barton","age":34,"gender":"F","address":"406 Seabring Street","employer":"Providco","email":"patriciabarton@providco.com","city":"Avoca","state":"RI","_is_real":true} +{"index":{"_id":"848"}} +{"account_number":848,"balance":15443,"firstname":"Carmella","lastname":"Cash","age":38,"gender":"M","address":"988 Exeter Street","employer":"Bristo","email":"carmellacash@bristo.com","city":"Northridge","state":"ID","_is_real":true} +{"index":{"_id":"850"}} +{"account_number":850,"balance":6531,"firstname":"Carlene","lastname":"Gaines","age":37,"gender":"F","address":"753 Monroe Place","employer":"Naxdis","email":"carlenegaines@naxdis.com","city":"Genoa","state":"OR","_is_real":true} +{"index":{"_id":"855"}} +{"account_number":855,"balance":40170,"firstname":"Mia","lastname":"Stevens","age":31,"gender":"F","address":"326 Driggs Avenue","employer":"Aeora","email":"miastevens@aeora.com","city":"Delwood","state":"IL","_is_real":true} +{"index":{"_id":"862"}} +{"account_number":862,"balance":38792,"firstname":"Clayton","lastname":"Golden","age":38,"gender":"F","address":"620 Regent Place","employer":"Accusage","email":"claytongolden@accusage.com","city":"Ona","state":"NC","_is_real":true} +{"index":{"_id":"867"}} +{"account_number":867,"balance":45453,"firstname":"Blanca","lastname":"Ellison","age":23,"gender":"F","address":"593 McKibben Street","employer":"Koogle","email":"blancaellison@koogle.com","city":"Frystown","state":"WY","_is_real":true} +{"index":{"_id":"874"}} +{"account_number":874,"balance":23079,"firstname":"Lynette","lastname":"Higgins","age":22,"gender":"M","address":"377 McKinley Avenue","employer":"Menbrain","email":"lynettehiggins@menbrain.com","city":"Manitou","state":"TX","_is_real":true} +{"index":{"_id":"879"}} +{"account_number":879,"balance":48332,"firstname":"Sabrina","lastname":"Lancaster","age":31,"gender":"F","address":"382 Oak Street","employer":"Webiotic","email":"sabrinalancaster@webiotic.com","city":"Lindisfarne","state":"AZ","_is_real":true} +{"index":{"_id":"881"}} +{"account_number":881,"balance":26684,"firstname":"Barnes","lastname":"Ware","age":38,"gender":"F","address":"666 Hooper Street","employer":"Norali","email":"barnesware@norali.com","city":"Cazadero","state":"GA","_is_real":true} +{"index":{"_id":"886"}} +{"account_number":886,"balance":14867,"firstname":"Willa","lastname":"Leblanc","age":38,"gender":"F","address":"773 Bergen Street","employer":"Nurali","email":"willaleblanc@nurali.com","city":"Hilltop","state":"NC","_is_real":true} +{"index":{"_id":"893"}} +{"account_number":893,"balance":42584,"firstname":"Moses","lastname":"Campos","age":38,"gender":"F","address":"991 Bevy Court","employer":"Trollery","email":"mosescampos@trollery.com","city":"Freetown","state":"AK","_is_real":true} +{"index":{"_id":"898"}} +{"account_number":898,"balance":12019,"firstname":"Lori","lastname":"Stevenson","age":29,"gender":"M","address":"910 Coles Street","employer":"Honotron","email":"loristevenson@honotron.com","city":"Shindler","state":"VT","_is_real":true} +{"index":{"_id":"901"}} +{"account_number":901,"balance":35038,"firstname":"Irma","lastname":"Dotson","age":23,"gender":"F","address":"245 Mayfair Drive","employer":"Bleeko","email":"irmadotson@bleeko.com","city":"Lodoga","state":"UT","_is_real":true} +{"index":{"_id":"906"}} +{"account_number":906,"balance":24073,"firstname":"Vicki","lastname":"Suarez","age":36,"gender":"M","address":"829 Roosevelt Place","employer":"Utara","email":"vickisuarez@utara.com","city":"Albrightsville","state":"AR","_is_real":true} +{"index":{"_id":"913"}} +{"account_number":913,"balance":47657,"firstname":"Margery","lastname":"Monroe","age":25,"gender":"M","address":"941 Fanchon Place","employer":"Exerta","email":"margerymonroe@exerta.com","city":"Bannock","state":"MD","_is_real":true} +{"index":{"_id":"918"}} +{"account_number":918,"balance":36776,"firstname":"Dianna","lastname":"Hernandez","age":25,"gender":"M","address":"499 Moultrie Street","employer":"Isologica","email":"diannahernandez@isologica.com","city":"Falconaire","state":"ID","_is_real":true} +{"index":{"_id":"920"}} +{"account_number":920,"balance":41513,"firstname":"Jerri","lastname":"Mitchell","age":26,"gender":"M","address":"831 Kent Street","employer":"Tasmania","email":"jerrimitchell@tasmania.com","city":"Cotopaxi","state":"IA","_is_real":true} +{"index":{"_id":"925"}} +{"account_number":925,"balance":18295,"firstname":"Rosario","lastname":"Jackson","age":24,"gender":"M","address":"178 Leonora Court","employer":"Progenex","email":"rosariojackson@progenex.com","city":"Rivereno","state":"DE","_is_real":true} +{"index":{"_id":"932"}} +{"account_number":932,"balance":3111,"firstname":"Summer","lastname":"Porter","age":33,"gender":"F","address":"949 Grand Avenue","employer":"Multiflex","email":"summerporter@multiflex.com","city":"Spokane","state":"OK","_is_real":true} +{"index":{"_id":"937"}} +{"account_number":937,"balance":43491,"firstname":"Selma","lastname":"Anderson","age":24,"gender":"M","address":"205 Reed Street","employer":"Dadabase","email":"selmaanderson@dadabase.com","city":"Malo","state":"AL","_is_real":true} +{"index":{"_id":"944"}} +{"account_number":944,"balance":46478,"firstname":"Donaldson","lastname":"Woodard","age":38,"gender":"F","address":"498 Laurel Avenue","employer":"Zogak","email":"donaldsonwoodard@zogak.com","city":"Hasty","state":"ID","_is_real":true} +{"index":{"_id":"949"}} +{"account_number":949,"balance":48703,"firstname":"Latasha","lastname":"Mullins","age":29,"gender":"F","address":"272 Lefferts Place","employer":"Zenolux","email":"latashamullins@zenolux.com","city":"Kieler","state":"MN","_is_real":true} +{"index":{"_id":"951"}} +{"account_number":951,"balance":36337,"firstname":"Tran","lastname":"Burris","age":25,"gender":"F","address":"561 Rutland Road","employer":"Geoform","email":"tranburris@geoform.com","city":"Longbranch","state":"IL","_is_real":true} +{"index":{"_id":"956"}} +{"account_number":956,"balance":19477,"firstname":"Randall","lastname":"Lynch","age":22,"gender":"F","address":"490 Madison Place","employer":"Cosmetex","email":"randalllynch@cosmetex.com","city":"Wells","state":"SD","_is_real":true} +{"index":{"_id":"963"}} +{"account_number":963,"balance":30461,"firstname":"Griffin","lastname":"Sheppard","age":20,"gender":"M","address":"682 Linden Street","employer":"Zanymax","email":"griffinsheppard@zanymax.com","city":"Fannett","state":"NM","_is_real":true} +{"index":{"_id":"968"}} +{"account_number":968,"balance":32371,"firstname":"Luella","lastname":"Burch","age":39,"gender":"M","address":"684 Arkansas Drive","employer":"Krag","email":"luellaburch@krag.com","city":"Brambleton","state":"SD","_is_real":true} +{"index":{"_id":"970"}} +{"account_number":970,"balance":19648,"firstname":"Forbes","lastname":"Wallace","age":28,"gender":"M","address":"990 Mill Road","employer":"Pheast","email":"forbeswallace@pheast.com","city":"Lopezo","state":"AK","_is_real":true} +{"index":{"_id":"975"}} +{"account_number":975,"balance":5239,"firstname":"Delores","lastname":"Booker","age":27,"gender":"F","address":"328 Conselyea Street","employer":"Centice","email":"deloresbooker@centice.com","city":"Williams","state":"HI","_is_real":true} +{"index":{"_id":"982"}} +{"account_number":982,"balance":16511,"firstname":"Buck","lastname":"Robinson","age":24,"gender":"M","address":"301 Melrose Street","employer":"Calcu","email":"buckrobinson@calcu.com","city":"Welch","state":"PA","_is_real":true} +{"index":{"_id":"987"}} +{"account_number":987,"balance":4072,"firstname":"Brock","lastname":"Sandoval","age":20,"gender":"F","address":"977 Gem Street","employer":"Fiberox","email":"brocksandoval@fiberox.com","city":"Celeryville","state":"NY","_is_real":true} +{"index":{"_id":"994"}} +{"account_number":994,"balance":33298,"firstname":"Madge","lastname":"Holcomb","age":31,"gender":"M","address":"612 Hawthorne Street","employer":"Escenta","email":"madgeholcomb@escenta.com","city":"Alafaya","state":"OR","_is_real":true} +{"index":{"_id":"999"}} +{"account_number":999,"balance":6087,"firstname":"Dorothy","lastname":"Barron","age":22,"gender":"F","address":"499 Laurel Avenue","employer":"Xurban","email":"dorothybarron@xurban.com","city":"Belvoir","state":"CA","_is_real":true} +{"index":{"_id":"4"}} +{"account_number":4,"balance":27658,"firstname":"Rodriquez","lastname":"Flores","age":31,"gender":"F","address":"986 Wyckoff Avenue","employer":"Tourmania","email":"rodriquezflores@tourmania.com","city":"Eastvale","state":"HI","_is_real":true} +{"index":{"_id":"9"}} +{"account_number":9,"balance":24776,"firstname":"Opal","lastname":"Meadows","age":39,"gender":"M","address":"963 Neptune Avenue","employer":"Cedward","email":"opalmeadows@cedward.com","city":"Olney","state":"OH","_is_real":true} +{"index":{"_id":"11"}} +{"account_number":11,"balance":20203,"firstname":"Jenkins","lastname":"Haney","age":20,"gender":"M","address":"740 Ferry Place","employer":"Qimonk","email":"jenkinshaney@qimonk.com","city":"Steinhatchee","state":"GA","_is_real":true} +{"index":{"_id":"16"}} +{"account_number":16,"balance":35883,"firstname":"Adrian","lastname":"Pitts","age":34,"gender":"F","address":"963 Fay Court","employer":"Combogene","email":"adrianpitts@combogene.com","city":"Remington","state":"SD","_is_real":true} +{"index":{"_id":"23"}} +{"account_number":23,"balance":42374,"firstname":"Kirsten","lastname":"Fox","age":20,"gender":"M","address":"330 Dumont Avenue","employer":"Codax","email":"kirstenfox@codax.com","city":"Walton","state":"AK","_is_real":true} +{"index":{"_id":"28"}} +{"account_number":28,"balance":42112,"firstname":"Vega","lastname":"Flynn","age":20,"gender":"M","address":"647 Hyman Court","employer":"Accupharm","email":"vegaflynn@accupharm.com","city":"Masthope","state":"OH","_is_real":true} +{"index":{"_id":"30"}} +{"account_number":30,"balance":19087,"firstname":"Lamb","lastname":"Townsend","age":26,"gender":"M","address":"169 Lyme Avenue","employer":"Geeknet","email":"lambtownsend@geeknet.com","city":"Epworth","state":"AL","_is_real":true} +{"index":{"_id":"35"}} +{"account_number":35,"balance":42039,"firstname":"Darla","lastname":"Bridges","age":27,"gender":"F","address":"315 Central Avenue","employer":"Xeronk","email":"darlabridges@xeronk.com","city":"Woodlake","state":"RI","_is_real":true} +{"index":{"_id":"42"}} +{"account_number":42,"balance":21137,"firstname":"Harding","lastname":"Hobbs","age":26,"gender":"F","address":"474 Ridgewood Place","employer":"Xth","email":"hardinghobbs@xth.com","city":"Heil","state":"ND","_is_real":true} +{"index":{"_id":"47"}} +{"account_number":47,"balance":33044,"firstname":"Georgia","lastname":"Wilkerson","age":23,"gender":"M","address":"369 Herbert Street","employer":"Endipin","email":"georgiawilkerson@endipin.com","city":"Dellview","state":"WI","_is_real":true} +{"index":{"_id":"54"}} +{"account_number":54,"balance":23406,"firstname":"Angel","lastname":"Mann","age":22,"gender":"F","address":"229 Ferris Street","employer":"Amtas","email":"angelmann@amtas.com","city":"Calverton","state":"WA","_is_real":true} +{"index":{"_id":"59"}} +{"account_number":59,"balance":37728,"firstname":"Malone","lastname":"Justice","age":37,"gender":"F","address":"721 Russell Street","employer":"Emoltra","email":"malonejustice@emoltra.com","city":"Trucksville","state":"HI","_is_real":true} +{"index":{"_id":"61"}} +{"account_number":61,"balance":6856,"firstname":"Shawn","lastname":"Baird","age":20,"gender":"M","address":"605 Monument Walk","employer":"Moltonic","email":"shawnbaird@moltonic.com","city":"Darlington","state":"MN","_is_real":true} +{"index":{"_id":"66"}} +{"account_number":66,"balance":25939,"firstname":"Franks","lastname":"Salinas","age":28,"gender":"M","address":"437 Hamilton Walk","employer":"Cowtown","email":"frankssalinas@cowtown.com","city":"Chase","state":"VT","_is_real":true} +{"index":{"_id":"73"}} +{"account_number":73,"balance":33457,"firstname":"Irene","lastname":"Stephenson","age":32,"gender":"M","address":"684 Miller Avenue","employer":"Hawkster","email":"irenestephenson@hawkster.com","city":"Levant","state":"AR","_is_real":true} +{"index":{"_id":"78"}} +{"account_number":78,"balance":48656,"firstname":"Elvira","lastname":"Patterson","age":23,"gender":"F","address":"834 Amber Street","employer":"Assistix","email":"elvirapatterson@assistix.com","city":"Dunbar","state":"TN","_is_real":true} +{"index":{"_id":"80"}} +{"account_number":80,"balance":13445,"firstname":"Lacey","lastname":"Blanchard","age":30,"gender":"F","address":"823 Himrod Street","employer":"Comdom","email":"laceyblanchard@comdom.com","city":"Matthews","state":"MO","_is_real":true} +{"index":{"_id":"85"}} +{"account_number":85,"balance":48735,"firstname":"Wilcox","lastname":"Sellers","age":20,"gender":"M","address":"212 Irving Avenue","employer":"Confrenzy","email":"wilcoxsellers@confrenzy.com","city":"Kipp","state":"MT","_is_real":true} +{"index":{"_id":"92"}} +{"account_number":92,"balance":26753,"firstname":"Gay","lastname":"Brewer","age":34,"gender":"M","address":"369 Ditmars Street","employer":"Savvy","email":"gaybrewer@savvy.com","city":"Moquino","state":"HI","_is_real":true} +{"index":{"_id":"97"}} +{"account_number":97,"balance":49671,"firstname":"Karen","lastname":"Trujillo","age":40,"gender":"F","address":"512 Cumberland Walk","employer":"Tsunamia","email":"karentrujillo@tsunamia.com","city":"Fredericktown","state":"MO","_is_real":true} +{"index":{"_id":"100"}} +{"account_number":100,"balance":29869,"firstname":"Madden","lastname":"Woods","age":32,"gender":"F","address":"696 Ryder Avenue","employer":"Slumberia","email":"maddenwoods@slumberia.com","city":"Deercroft","state":"ME","_is_real":true} +{"index":{"_id":"105"}} +{"account_number":105,"balance":29654,"firstname":"Castillo","lastname":"Dickerson","age":33,"gender":"F","address":"673 Oxford Street","employer":"Tellifly","email":"castillodickerson@tellifly.com","city":"Succasunna","state":"NY","_is_real":true} +{"index":{"_id":"112"}} +{"account_number":112,"balance":38395,"firstname":"Frederick","lastname":"Case","age":30,"gender":"F","address":"580 Lexington Avenue","employer":"Talkalot","email":"frederickcase@talkalot.com","city":"Orovada","state":"MA","_is_real":true} +{"index":{"_id":"117"}} +{"account_number":117,"balance":48831,"firstname":"Robin","lastname":"Hays","age":38,"gender":"F","address":"347 Hornell Loop","employer":"Pasturia","email":"robinhays@pasturia.com","city":"Sims","state":"WY","_is_real":true} +{"index":{"_id":"124"}} +{"account_number":124,"balance":16425,"firstname":"Fern","lastname":"Lambert","age":20,"gender":"M","address":"511 Jay Street","employer":"Furnitech","email":"fernlambert@furnitech.com","city":"Cloverdale","state":"FL","_is_real":true} +{"index":{"_id":"129"}} +{"account_number":129,"balance":42409,"firstname":"Alexandria","lastname":"Sanford","age":33,"gender":"F","address":"934 Ridgecrest Terrace","employer":"Kyagoro","email":"alexandriasanford@kyagoro.com","city":"Concho","state":"UT","_is_real":true} +{"index":{"_id":"131"}} +{"account_number":131,"balance":28030,"firstname":"Dollie","lastname":"Koch","age":22,"gender":"F","address":"287 Manhattan Avenue","employer":"Skinserve","email":"dolliekoch@skinserve.com","city":"Shasta","state":"PA","_is_real":true} +{"index":{"_id":"136"}} +{"account_number":136,"balance":45801,"firstname":"Winnie","lastname":"Holland","age":38,"gender":"M","address":"198 Mill Lane","employer":"Neteria","email":"winnieholland@neteria.com","city":"Urie","state":"IL","_is_real":true} +{"index":{"_id":"143"}} +{"account_number":143,"balance":43093,"firstname":"Cohen","lastname":"Noble","age":39,"gender":"M","address":"454 Nelson Street","employer":"Buzzworks","email":"cohennoble@buzzworks.com","city":"Norvelt","state":"CO","_is_real":true} +{"index":{"_id":"148"}} +{"account_number":148,"balance":3662,"firstname":"Annmarie","lastname":"Snider","age":34,"gender":"F","address":"857 Lafayette Walk","employer":"Edecine","email":"annmariesnider@edecine.com","city":"Hollins","state":"OH","_is_real":true} +{"index":{"_id":"150"}} +{"account_number":150,"balance":15306,"firstname":"Ortega","lastname":"Dalton","age":20,"gender":"M","address":"237 Mermaid Avenue","employer":"Rameon","email":"ortegadalton@rameon.com","city":"Maxville","state":"NH","_is_real":true} +{"index":{"_id":"155"}} +{"account_number":155,"balance":27878,"firstname":"Atkinson","lastname":"Hudson","age":39,"gender":"F","address":"434 Colin Place","employer":"Qualitern","email":"atkinsonhudson@qualitern.com","city":"Hoehne","state":"OH","_is_real":true} +{"index":{"_id":"162"}} +{"account_number":162,"balance":6302,"firstname":"Griffith","lastname":"Calderon","age":35,"gender":"M","address":"871 Vandervoort Place","employer":"Quotezart","email":"griffithcalderon@quotezart.com","city":"Barclay","state":"FL","_is_real":true} +{"index":{"_id":"167"}} +{"account_number":167,"balance":42051,"firstname":"Hampton","lastname":"Ryan","age":20,"gender":"M","address":"618 Fleet Place","employer":"Zipak","email":"hamptonryan@zipak.com","city":"Irwin","state":"KS","_is_real":true} +{"index":{"_id":"174"}} +{"account_number":174,"balance":1464,"firstname":"Gamble","lastname":"Pierce","age":23,"gender":"F","address":"650 Eagle Street","employer":"Matrixity","email":"gamblepierce@matrixity.com","city":"Abiquiu","state":"OR","_is_real":true} +{"index":{"_id":"179"}} +{"account_number":179,"balance":13265,"firstname":"Elise","lastname":"Drake","age":25,"gender":"M","address":"305 Christopher Avenue","employer":"Turnling","email":"elisedrake@turnling.com","city":"Loretto","state":"LA","_is_real":true} +{"index":{"_id":"181"}} +{"account_number":181,"balance":27983,"firstname":"Bennett","lastname":"Hampton","age":22,"gender":"F","address":"435 Billings Place","employer":"Voipa","email":"bennetthampton@voipa.com","city":"Rodman","state":"WY","_is_real":true} +{"index":{"_id":"186"}} +{"account_number":186,"balance":18373,"firstname":"Kline","lastname":"Joyce","age":32,"gender":"M","address":"285 Falmouth Street","employer":"Tetratrex","email":"klinejoyce@tetratrex.com","city":"Klondike","state":"SD","_is_real":true} +{"index":{"_id":"193"}} +{"account_number":193,"balance":13412,"firstname":"Patty","lastname":"Petty","age":34,"gender":"F","address":"251 Vermont Street","employer":"Kinetica","email":"pattypetty@kinetica.com","city":"Grantville","state":"MS","_is_real":true} +{"index":{"_id":"198"}} +{"account_number":198,"balance":19686,"firstname":"Rachael","lastname":"Sharp","age":38,"gender":"F","address":"443 Vernon Avenue","employer":"Powernet","email":"rachaelsharp@powernet.com","city":"Canoochee","state":"UT","_is_real":true} +{"index":{"_id":"201"}} +{"account_number":201,"balance":14586,"firstname":"Ronda","lastname":"Perry","age":25,"gender":"F","address":"856 Downing Street","employer":"Artiq","email":"rondaperry@artiq.com","city":"Colton","state":"WV","_is_real":true} +{"index":{"_id":"206"}} +{"account_number":206,"balance":47423,"firstname":"Kelli","lastname":"Francis","age":20,"gender":"M","address":"671 George Street","employer":"Exoswitch","email":"kellifrancis@exoswitch.com","city":"Babb","state":"NJ","_is_real":true} +{"index":{"_id":"213"}} +{"account_number":213,"balance":34172,"firstname":"Bauer","lastname":"Summers","age":27,"gender":"M","address":"257 Boynton Place","employer":"Voratak","email":"bauersummers@voratak.com","city":"Oceola","state":"NC","_is_real":true} +{"index":{"_id":"218"}} +{"account_number":218,"balance":26702,"firstname":"Garrison","lastname":"Bryan","age":24,"gender":"F","address":"478 Greenpoint Avenue","employer":"Uniworld","email":"garrisonbryan@uniworld.com","city":"Comptche","state":"WI","_is_real":true} +{"index":{"_id":"220"}} +{"account_number":220,"balance":3086,"firstname":"Tania","lastname":"Middleton","age":22,"gender":"F","address":"541 Gunther Place","employer":"Zerology","email":"taniamiddleton@zerology.com","city":"Linwood","state":"IN","_is_real":true} +{"index":{"_id":"225"}} +{"account_number":225,"balance":21949,"firstname":"Maryann","lastname":"Murphy","age":24,"gender":"F","address":"894 Bridgewater Street","employer":"Cinesanct","email":"maryannmurphy@cinesanct.com","city":"Cartwright","state":"RI","_is_real":true} +{"index":{"_id":"232"}} +{"account_number":232,"balance":11984,"firstname":"Carr","lastname":"Jensen","age":34,"gender":"F","address":"995 Micieli Place","employer":"Biohab","email":"carrjensen@biohab.com","city":"Waikele","state":"OH","_is_real":true} +{"index":{"_id":"237"}} +{"account_number":237,"balance":5603,"firstname":"Kirby","lastname":"Watkins","age":27,"gender":"F","address":"348 Blake Court","employer":"Sonique","email":"kirbywatkins@sonique.com","city":"Freelandville","state":"PA","_is_real":true} +{"index":{"_id":"244"}} +{"account_number":244,"balance":8048,"firstname":"Judith","lastname":"Riggs","age":27,"gender":"F","address":"590 Kosciusko Street","employer":"Arctiq","email":"judithriggs@arctiq.com","city":"Gorham","state":"DC","_is_real":true} +{"index":{"_id":"249"}} +{"account_number":249,"balance":16822,"firstname":"Mckinney","lastname":"Gallagher","age":38,"gender":"F","address":"939 Seigel Court","employer":"Premiant","email":"mckinneygallagher@premiant.com","city":"Catharine","state":"NH","_is_real":true} +{"index":{"_id":"251"}} +{"account_number":251,"balance":13475,"firstname":"Marks","lastname":"Graves","age":39,"gender":"F","address":"427 Lawn Court","employer":"Dentrex","email":"marksgraves@dentrex.com","city":"Waukeenah","state":"IL","_is_real":true} +{"index":{"_id":"256"}} +{"account_number":256,"balance":48318,"firstname":"Simon","lastname":"Hogan","age":31,"gender":"M","address":"789 Suydam Place","employer":"Dancerity","email":"simonhogan@dancerity.com","city":"Dargan","state":"GA","_is_real":true} +{"index":{"_id":"263"}} +{"account_number":263,"balance":12837,"firstname":"Thornton","lastname":"Meyer","age":29,"gender":"M","address":"575 Elliott Place","employer":"Peticular","email":"thorntonmeyer@peticular.com","city":"Dotsero","state":"NH","_is_real":true} +{"index":{"_id":"268"}} +{"account_number":268,"balance":20925,"firstname":"Avis","lastname":"Blackwell","age":36,"gender":"M","address":"569 Jerome Avenue","employer":"Magnina","email":"avisblackwell@magnina.com","city":"Bethany","state":"MD","_is_real":true} +{"index":{"_id":"270"}} +{"account_number":270,"balance":43951,"firstname":"Moody","lastname":"Harmon","age":39,"gender":"F","address":"233 Vanderbilt Street","employer":"Otherside","email":"moodyharmon@otherside.com","city":"Elwood","state":"MT","_is_real":true} +{"index":{"_id":"275"}} +{"account_number":275,"balance":2384,"firstname":"Reynolds","lastname":"Barnett","age":31,"gender":"M","address":"394 Stockton Street","employer":"Austex","email":"reynoldsbarnett@austex.com","city":"Grandview","state":"MS","_is_real":true} +{"index":{"_id":"282"}} +{"account_number":282,"balance":38540,"firstname":"Gay","lastname":"Schultz","age":25,"gender":"F","address":"805 Claver Place","employer":"Handshake","email":"gayschultz@handshake.com","city":"Tampico","state":"MA","_is_real":true} +{"index":{"_id":"287"}} +{"account_number":287,"balance":10845,"firstname":"Valerie","lastname":"Lang","age":35,"gender":"F","address":"423 Midwood Street","employer":"Quarx","email":"valerielang@quarx.com","city":"Cannondale","state":"VT","_is_real":true} +{"index":{"_id":"294"}} +{"account_number":294,"balance":29582,"firstname":"Pitts","lastname":"Haynes","age":26,"gender":"M","address":"901 Broome Street","employer":"Aquazure","email":"pittshaynes@aquazure.com","city":"Turah","state":"SD","_is_real":true} +{"index":{"_id":"299"}} +{"account_number":299,"balance":40825,"firstname":"Angela","lastname":"Talley","age":36,"gender":"F","address":"822 Bills Place","employer":"Remold","email":"angelatalley@remold.com","city":"Bethpage","state":"DC","_is_real":true} +{"index":{"_id":"302"}} +{"account_number":302,"balance":11298,"firstname":"Isabella","lastname":"Hewitt","age":40,"gender":"M","address":"455 Bedford Avenue","employer":"Cincyr","email":"isabellahewitt@cincyr.com","city":"Blanford","state":"IN","_is_real":true} +{"index":{"_id":"307"}} +{"account_number":307,"balance":43355,"firstname":"Enid","lastname":"Ashley","age":23,"gender":"M","address":"412 Emerson Place","employer":"Avenetro","email":"enidashley@avenetro.com","city":"Catherine","state":"WI","_is_real":true} +{"index":{"_id":"314"}} +{"account_number":314,"balance":5848,"firstname":"Norton","lastname":"Norton","age":35,"gender":"M","address":"252 Ditmas Avenue","employer":"Talkola","email":"nortonnorton@talkola.com","city":"Veyo","state":"SC","_is_real":true} +{"index":{"_id":"319"}} +{"account_number":319,"balance":15430,"firstname":"Ferrell","lastname":"Mckinney","age":36,"gender":"M","address":"874 Cranberry Street","employer":"Portaline","email":"ferrellmckinney@portaline.com","city":"Rose","state":"WV","_is_real":true} +{"index":{"_id":"321"}} +{"account_number":321,"balance":43370,"firstname":"Marta","lastname":"Larsen","age":35,"gender":"M","address":"617 Williams Court","employer":"Manufact","email":"martalarsen@manufact.com","city":"Sisquoc","state":"MA","_is_real":true} +{"index":{"_id":"326"}} +{"account_number":326,"balance":9692,"firstname":"Pearl","lastname":"Reese","age":30,"gender":"F","address":"451 Colonial Court","employer":"Accruex","email":"pearlreese@accruex.com","city":"Westmoreland","state":"MD","_is_real":true} +{"index":{"_id":"333"}} +{"account_number":333,"balance":22778,"firstname":"Trudy","lastname":"Sweet","age":27,"gender":"F","address":"881 Kiely Place","employer":"Acumentor","email":"trudysweet@acumentor.com","city":"Kent","state":"IA","_is_real":true} +{"index":{"_id":"338"}} +{"account_number":338,"balance":6969,"firstname":"Pierce","lastname":"Lawrence","age":35,"gender":"M","address":"318 Gallatin Place","employer":"Lunchpad","email":"piercelawrence@lunchpad.com","city":"Iola","state":"MD","_is_real":true} +{"index":{"_id":"340"}} +{"account_number":340,"balance":42072,"firstname":"Juarez","lastname":"Gutierrez","age":40,"gender":"F","address":"802 Seba Avenue","employer":"Billmed","email":"juarezgutierrez@billmed.com","city":"Malott","state":"OH","_is_real":true} +{"index":{"_id":"345"}} +{"account_number":345,"balance":9812,"firstname":"Parker","lastname":"Hines","age":38,"gender":"M","address":"715 Mill Avenue","employer":"Baluba","email":"parkerhines@baluba.com","city":"Blackgum","state":"KY","_is_real":true} +{"index":{"_id":"352"}} +{"account_number":352,"balance":20290,"firstname":"Kendra","lastname":"Mcintosh","age":31,"gender":"F","address":"963 Wolf Place","employer":"Orboid","email":"kendramcintosh@orboid.com","city":"Bladensburg","state":"AK","_is_real":true} +{"index":{"_id":"357"}} +{"account_number":357,"balance":15102,"firstname":"Adele","lastname":"Carroll","age":39,"gender":"F","address":"381 Arion Place","employer":"Aquafire","email":"adelecarroll@aquafire.com","city":"Springville","state":"RI","_is_real":true} +{"index":{"_id":"364"}} +{"account_number":364,"balance":35247,"firstname":"Felicia","lastname":"Merrill","age":40,"gender":"F","address":"229 Branton Street","employer":"Prosely","email":"feliciamerrill@prosely.com","city":"Dola","state":"MA","_is_real":true} +{"index":{"_id":"369"}} +{"account_number":369,"balance":17047,"firstname":"Mcfadden","lastname":"Guy","age":28,"gender":"F","address":"445 Lott Avenue","employer":"Kangle","email":"mcfaddenguy@kangle.com","city":"Greenbackville","state":"DE","_is_real":true} +{"index":{"_id":"371"}} +{"account_number":371,"balance":19751,"firstname":"Barker","lastname":"Allen","age":32,"gender":"F","address":"295 Wallabout Street","employer":"Nexgene","email":"barkerallen@nexgene.com","city":"Nanafalia","state":"NE","_is_real":true} +{"index":{"_id":"376"}} +{"account_number":376,"balance":44407,"firstname":"Mcmillan","lastname":"Dunn","age":21,"gender":"F","address":"771 Dorchester Road","employer":"Eargo","email":"mcmillandunn@eargo.com","city":"Yogaville","state":"RI","_is_real":true} +{"index":{"_id":"383"}} +{"account_number":383,"balance":48889,"firstname":"Knox","lastname":"Larson","age":28,"gender":"F","address":"962 Bartlett Place","employer":"Bostonic","email":"knoxlarson@bostonic.com","city":"Smeltertown","state":"TX","_is_real":true} +{"index":{"_id":"388"}} +{"account_number":388,"balance":9606,"firstname":"Julianne","lastname":"Nicholson","age":26,"gender":"F","address":"338 Crescent Street","employer":"Viasia","email":"juliannenicholson@viasia.com","city":"Alleghenyville","state":"MO","_is_real":true} +{"index":{"_id":"390"}} +{"account_number":390,"balance":7464,"firstname":"Ramona","lastname":"Roy","age":32,"gender":"M","address":"135 Banner Avenue","employer":"Deminimum","email":"ramonaroy@deminimum.com","city":"Dodge","state":"ID","_is_real":true} +{"index":{"_id":"395"}} +{"account_number":395,"balance":18679,"firstname":"Juliet","lastname":"Whitaker","age":31,"gender":"M","address":"128 Remsen Avenue","employer":"Toyletry","email":"julietwhitaker@toyletry.com","city":"Yonah","state":"LA","_is_real":true} +{"index":{"_id":"403"}} +{"account_number":403,"balance":18833,"firstname":"Williamson","lastname":"Horn","age":32,"gender":"M","address":"223 Strickland Avenue","employer":"Nimon","email":"williamsonhorn@nimon.com","city":"Bawcomville","state":"NJ","_is_real":true} +{"index":{"_id":"408"}} +{"account_number":408,"balance":34666,"firstname":"Lidia","lastname":"Guerrero","age":30,"gender":"M","address":"254 Stratford Road","employer":"Snowpoke","email":"lidiaguerrero@snowpoke.com","city":"Fairlee","state":"LA","_is_real":true} +{"index":{"_id":"410"}} +{"account_number":410,"balance":31200,"firstname":"Fox","lastname":"Cardenas","age":39,"gender":"M","address":"987 Monitor Street","employer":"Corpulse","email":"foxcardenas@corpulse.com","city":"Southview","state":"NE","_is_real":true} +{"index":{"_id":"415"}} +{"account_number":415,"balance":19449,"firstname":"Martinez","lastname":"Benson","age":36,"gender":"M","address":"172 Berkeley Place","employer":"Enersol","email":"martinezbenson@enersol.com","city":"Chumuckla","state":"AL","_is_real":true} +{"index":{"_id":"422"}} +{"account_number":422,"balance":40162,"firstname":"Brigitte","lastname":"Scott","age":26,"gender":"M","address":"662 Vermont Court","employer":"Waretel","email":"brigittescott@waretel.com","city":"Elrama","state":"VA","_is_real":true} +{"index":{"_id":"427"}} +{"account_number":427,"balance":1463,"firstname":"Rebekah","lastname":"Garrison","age":36,"gender":"F","address":"837 Hampton Avenue","employer":"Niquent","email":"rebekahgarrison@niquent.com","city":"Zarephath","state":"NY","_is_real":true} +{"index":{"_id":"434"}} +{"account_number":434,"balance":11329,"firstname":"Christa","lastname":"Huff","age":25,"gender":"M","address":"454 Oriental Boulevard","employer":"Earthpure","email":"christahuff@earthpure.com","city":"Stevens","state":"DC","_is_real":true} +{"index":{"_id":"439"}} +{"account_number":439,"balance":22752,"firstname":"Lula","lastname":"Williams","age":35,"gender":"M","address":"630 Furman Avenue","employer":"Vinch","email":"lulawilliams@vinch.com","city":"Newcastle","state":"ME","_is_real":true} +{"index":{"_id":"441"}} +{"account_number":441,"balance":47947,"firstname":"Dickson","lastname":"Mcgee","age":29,"gender":"M","address":"478 Knight Court","employer":"Gogol","email":"dicksonmcgee@gogol.com","city":"Laurelton","state":"AR","_is_real":true} +{"index":{"_id":"446"}} +{"account_number":446,"balance":23071,"firstname":"Lolita","lastname":"Fleming","age":32,"gender":"F","address":"918 Bridge Street","employer":"Vidto","email":"lolitafleming@vidto.com","city":"Brownlee","state":"HI","_is_real":true} +{"index":{"_id":"453"}} +{"account_number":453,"balance":21520,"firstname":"Hood","lastname":"Powell","age":24,"gender":"F","address":"479 Brevoort Place","employer":"Vortexaco","email":"hoodpowell@vortexaco.com","city":"Alderpoint","state":"CT","_is_real":true} +{"index":{"_id":"458"}} +{"account_number":458,"balance":8865,"firstname":"Aida","lastname":"Wolf","age":21,"gender":"F","address":"403 Thames Street","employer":"Isis","email":"aidawolf@isis.com","city":"Bordelonville","state":"ME","_is_real":true} +{"index":{"_id":"460"}} +{"account_number":460,"balance":37734,"firstname":"Aguirre","lastname":"White","age":21,"gender":"F","address":"190 Crooke Avenue","employer":"Unq","email":"aguirrewhite@unq.com","city":"Albany","state":"NJ","_is_real":true} +{"index":{"_id":"465"}} +{"account_number":465,"balance":10681,"firstname":"Pearlie","lastname":"Holman","age":29,"gender":"M","address":"916 Evergreen Avenue","employer":"Hometown","email":"pearlieholman@hometown.com","city":"Needmore","state":"UT","_is_real":true} +{"index":{"_id":"472"}} +{"account_number":472,"balance":25571,"firstname":"Lee","lastname":"Long","age":32,"gender":"F","address":"288 Mill Street","employer":"Comverges","email":"leelong@comverges.com","city":"Movico","state":"MT","_is_real":true} +{"index":{"_id":"477"}} +{"account_number":477,"balance":25892,"firstname":"Holcomb","lastname":"Cobb","age":40,"gender":"M","address":"369 Marconi Place","employer":"Steeltab","email":"holcombcobb@steeltab.com","city":"Byrnedale","state":"CA","_is_real":true} +{"index":{"_id":"484"}} +{"account_number":484,"balance":3274,"firstname":"Staci","lastname":"Melendez","age":35,"gender":"F","address":"751 Otsego Street","employer":"Namebox","email":"stacimelendez@namebox.com","city":"Harborton","state":"NV","_is_real":true} +{"index":{"_id":"489"}} +{"account_number":489,"balance":7879,"firstname":"Garrett","lastname":"Langley","age":36,"gender":"M","address":"331 Bowne Street","employer":"Zillidium","email":"garrettlangley@zillidium.com","city":"Riviera","state":"LA","_is_real":true} +{"index":{"_id":"491"}} +{"account_number":491,"balance":42942,"firstname":"Teresa","lastname":"Owen","age":24,"gender":"F","address":"713 Canton Court","employer":"Plasmos","email":"teresaowen@plasmos.com","city":"Bartonsville","state":"NH","_is_real":true} +{"index":{"_id":"496"}} +{"account_number":496,"balance":14869,"firstname":"Alison","lastname":"Conrad","age":35,"gender":"F","address":"347 Varet Street","employer":"Perkle","email":"alisonconrad@perkle.com","city":"Cliffside","state":"OH","_is_real":true} +{"index":{"_id":"504"}} +{"account_number":504,"balance":49205,"firstname":"Shanna","lastname":"Chambers","age":23,"gender":"M","address":"220 Beard Street","employer":"Corporana","email":"shannachambers@corporana.com","city":"Cashtown","state":"AZ","_is_real":true} +{"index":{"_id":"509"}} +{"account_number":509,"balance":34754,"firstname":"Durham","lastname":"Pacheco","age":40,"gender":"M","address":"129 Plymouth Street","employer":"Datacator","email":"durhampacheco@datacator.com","city":"Loveland","state":"NC","_is_real":true} +{"index":{"_id":"511"}} +{"account_number":511,"balance":40908,"firstname":"Elba","lastname":"Grant","age":24,"gender":"F","address":"157 Bijou Avenue","employer":"Dognost","email":"elbagrant@dognost.com","city":"Coyote","state":"MT","_is_real":true} +{"index":{"_id":"516"}} +{"account_number":516,"balance":44940,"firstname":"Roy","lastname":"Smith","age":37,"gender":"M","address":"770 Cherry Street","employer":"Parleynet","email":"roysmith@parleynet.com","city":"Carrsville","state":"RI","_is_real":true} +{"index":{"_id":"523"}} +{"account_number":523,"balance":28729,"firstname":"Amalia","lastname":"Benjamin","age":40,"gender":"F","address":"173 Bushwick Place","employer":"Sentia","email":"amaliabenjamin@sentia.com","city":"Jacumba","state":"OK","_is_real":true} +{"index":{"_id":"528"}} +{"account_number":528,"balance":4071,"firstname":"Thompson","lastname":"Hoover","age":27,"gender":"F","address":"580 Garden Street","employer":"Portalis","email":"thompsonhoover@portalis.com","city":"Knowlton","state":"AL","_is_real":true} +{"index":{"_id":"530"}} +{"account_number":530,"balance":8840,"firstname":"Kathrine","lastname":"Evans","age":37,"gender":"M","address":"422 Division Place","employer":"Spherix","email":"kathrineevans@spherix.com","city":"Biddle","state":"CO","_is_real":true} +{"index":{"_id":"535"}} +{"account_number":535,"balance":8715,"firstname":"Fry","lastname":"George","age":34,"gender":"M","address":"722 Green Street","employer":"Ewaves","email":"frygeorge@ewaves.com","city":"Kenmar","state":"DE","_is_real":true} +{"index":{"_id":"542"}} +{"account_number":542,"balance":23285,"firstname":"Michelle","lastname":"Mayo","age":35,"gender":"M","address":"657 Caton Place","employer":"Biflex","email":"michellemayo@biflex.com","city":"Beaverdale","state":"WY","_is_real":true} +{"index":{"_id":"547"}} +{"account_number":547,"balance":12870,"firstname":"Eaton","lastname":"Rios","age":32,"gender":"M","address":"744 Withers Street","employer":"Podunk","email":"eatonrios@podunk.com","city":"Chelsea","state":"IA","_is_real":true} +{"index":{"_id":"554"}} +{"account_number":554,"balance":33163,"firstname":"Townsend","lastname":"Atkins","age":39,"gender":"M","address":"566 Ira Court","employer":"Acruex","email":"townsendatkins@acruex.com","city":"Valle","state":"IA","_is_real":true} +{"index":{"_id":"559"}} +{"account_number":559,"balance":11450,"firstname":"Tonia","lastname":"Schmidt","age":38,"gender":"F","address":"508 Sheffield Avenue","employer":"Extro","email":"toniaschmidt@extro.com","city":"Newry","state":"CT","_is_real":true} +{"index":{"_id":"561"}} +{"account_number":561,"balance":12370,"firstname":"Sellers","lastname":"Davis","age":30,"gender":"M","address":"860 Madoc Avenue","employer":"Isodrive","email":"sellersdavis@isodrive.com","city":"Trail","state":"KS","_is_real":true} +{"index":{"_id":"566"}} +{"account_number":566,"balance":6183,"firstname":"Cox","lastname":"Roman","age":37,"gender":"M","address":"349 Winthrop Street","employer":"Medcom","email":"coxroman@medcom.com","city":"Rosewood","state":"WY","_is_real":true} +{"index":{"_id":"573"}} +{"account_number":573,"balance":32171,"firstname":"Callie","lastname":"Castaneda","age":36,"gender":"M","address":"799 Scott Avenue","employer":"Earthwax","email":"calliecastaneda@earthwax.com","city":"Marshall","state":"NH","_is_real":true} +{"index":{"_id":"578"}} +{"account_number":578,"balance":34259,"firstname":"Holmes","lastname":"Mcknight","age":37,"gender":"M","address":"969 Metropolitan Avenue","employer":"Cubicide","email":"holmesmcknight@cubicide.com","city":"Aguila","state":"PA","_is_real":true} +{"index":{"_id":"580"}} +{"account_number":580,"balance":13716,"firstname":"Mcmahon","lastname":"York","age":34,"gender":"M","address":"475 Beacon Court","employer":"Zillar","email":"mcmahonyork@zillar.com","city":"Farmington","state":"MO","_is_real":true} +{"index":{"_id":"585"}} +{"account_number":585,"balance":26745,"firstname":"Nieves","lastname":"Nolan","age":32,"gender":"M","address":"115 Seagate Terrace","employer":"Jumpstack","email":"nievesnolan@jumpstack.com","city":"Eastmont","state":"UT","_is_real":true} +{"index":{"_id":"592"}} +{"account_number":592,"balance":32968,"firstname":"Head","lastname":"Webster","age":36,"gender":"F","address":"987 Lefferts Avenue","employer":"Empirica","email":"headwebster@empirica.com","city":"Rockingham","state":"TN","_is_real":true} +{"index":{"_id":"597"}} +{"account_number":597,"balance":11246,"firstname":"Penny","lastname":"Knowles","age":33,"gender":"M","address":"139 Forbell Street","employer":"Ersum","email":"pennyknowles@ersum.com","city":"Vallonia","state":"IA","_is_real":true} +{"index":{"_id":"600"}} +{"account_number":600,"balance":10336,"firstname":"Simmons","lastname":"Byers","age":37,"gender":"M","address":"250 Dictum Court","employer":"Qualitex","email":"simmonsbyers@qualitex.com","city":"Wanship","state":"OH","_is_real":true} +{"index":{"_id":"605"}} +{"account_number":605,"balance":38427,"firstname":"Mcclain","lastname":"Manning","age":24,"gender":"M","address":"832 Leonard Street","employer":"Qiao","email":"mcclainmanning@qiao.com","city":"Calvary","state":"TX","_is_real":true} +{"index":{"_id":"612"}} +{"account_number":612,"balance":11868,"firstname":"Dunn","lastname":"Cameron","age":32,"gender":"F","address":"156 Lorimer Street","employer":"Isonus","email":"dunncameron@isonus.com","city":"Virgie","state":"ND","_is_real":true} +{"index":{"_id":"617"}} +{"account_number":617,"balance":35445,"firstname":"Kitty","lastname":"Cooley","age":22,"gender":"M","address":"788 Seagate Avenue","employer":"Ultrimax","email":"kittycooley@ultrimax.com","city":"Clarktown","state":"MD","_is_real":true} +{"index":{"_id":"624"}} +{"account_number":624,"balance":27538,"firstname":"Roxanne","lastname":"Franklin","age":39,"gender":"F","address":"299 Woodrow Court","employer":"Silodyne","email":"roxannefranklin@silodyne.com","city":"Roulette","state":"VA","_is_real":true} +{"index":{"_id":"629"}} +{"account_number":629,"balance":32987,"firstname":"Mcclure","lastname":"Rodgers","age":26,"gender":"M","address":"806 Pierrepont Place","employer":"Elita","email":"mcclurerodgers@elita.com","city":"Brownsville","state":"MI","_is_real":true} +{"index":{"_id":"631"}} +{"account_number":631,"balance":21657,"firstname":"Corrine","lastname":"Barber","age":32,"gender":"F","address":"447 Hunts Lane","employer":"Quarmony","email":"corrinebarber@quarmony.com","city":"Wyano","state":"IL","_is_real":true} +{"index":{"_id":"636"}} +{"account_number":636,"balance":8036,"firstname":"Agnes","lastname":"Hooper","age":25,"gender":"M","address":"865 Hanson Place","employer":"Digial","email":"agneshooper@digial.com","city":"Sperryville","state":"OK","_is_real":true} +{"index":{"_id":"643"}} +{"account_number":643,"balance":8057,"firstname":"Hendricks","lastname":"Stokes","age":23,"gender":"F","address":"142 Barbey Street","employer":"Remotion","email":"hendricksstokes@remotion.com","city":"Lewis","state":"MA","_is_real":true} +{"index":{"_id":"648"}} +{"account_number":648,"balance":11506,"firstname":"Terry","lastname":"Montgomery","age":21,"gender":"F","address":"115 Franklin Avenue","employer":"Enervate","email":"terrymontgomery@enervate.com","city":"Bascom","state":"MA","_is_real":true} +{"index":{"_id":"650"}} +{"account_number":650,"balance":18091,"firstname":"Benton","lastname":"Knight","age":28,"gender":"F","address":"850 Aitken Place","employer":"Pholio","email":"bentonknight@pholio.com","city":"Cobbtown","state":"AL","_is_real":true} +{"index":{"_id":"655"}} +{"account_number":655,"balance":22912,"firstname":"Eula","lastname":"Taylor","age":30,"gender":"M","address":"520 Orient Avenue","employer":"Miracula","email":"eulataylor@miracula.com","city":"Wacissa","state":"IN","_is_real":true} +{"index":{"_id":"662"}} +{"account_number":662,"balance":10138,"firstname":"Daisy","lastname":"Burnett","age":33,"gender":"M","address":"114 Norman Avenue","employer":"Liquicom","email":"daisyburnett@liquicom.com","city":"Grahamtown","state":"MD","_is_real":true} +{"index":{"_id":"667"}} +{"account_number":667,"balance":22559,"firstname":"Juliana","lastname":"Chase","age":32,"gender":"M","address":"496 Coleridge Street","employer":"Comtract","email":"julianachase@comtract.com","city":"Wilsonia","state":"NJ","_is_real":true} +{"index":{"_id":"674"}} +{"account_number":674,"balance":36038,"firstname":"Watts","lastname":"Shannon","age":22,"gender":"F","address":"600 Story Street","employer":"Joviold","email":"wattsshannon@joviold.com","city":"Fairhaven","state":"ID","_is_real":true} +{"index":{"_id":"679"}} +{"account_number":679,"balance":20149,"firstname":"Henrietta","lastname":"Bonner","age":33,"gender":"M","address":"461 Bond Street","employer":"Geekol","email":"henriettabonner@geekol.com","city":"Richville","state":"WA","_is_real":true} +{"index":{"_id":"681"}} +{"account_number":681,"balance":34244,"firstname":"Velazquez","lastname":"Wolfe","age":33,"gender":"M","address":"773 Eckford Street","employer":"Zisis","email":"velazquezwolfe@zisis.com","city":"Smock","state":"ME","_is_real":true} +{"index":{"_id":"686"}} +{"account_number":686,"balance":10116,"firstname":"Decker","lastname":"Mcclure","age":30,"gender":"F","address":"236 Commerce Street","employer":"Everest","email":"deckermcclure@everest.com","city":"Gibbsville","state":"TN","_is_real":true} +{"index":{"_id":"693"}} +{"account_number":693,"balance":31233,"firstname":"Tabatha","lastname":"Zimmerman","age":30,"gender":"F","address":"284 Emmons Avenue","employer":"Pushcart","email":"tabathazimmerman@pushcart.com","city":"Esmont","state":"NC","_is_real":true} +{"index":{"_id":"698"}} +{"account_number":698,"balance":14965,"firstname":"Baker","lastname":"Armstrong","age":36,"gender":"F","address":"796 Tehama Street","employer":"Nurplex","email":"bakerarmstrong@nurplex.com","city":"Starks","state":"UT","_is_real":true} +{"index":{"_id":"701"}} +{"account_number":701,"balance":23772,"firstname":"Gardner","lastname":"Griffith","age":27,"gender":"M","address":"187 Moore Place","employer":"Vertide","email":"gardnergriffith@vertide.com","city":"Coventry","state":"NV","_is_real":true} +{"index":{"_id":"706"}} +{"account_number":706,"balance":5282,"firstname":"Eliza","lastname":"Potter","age":39,"gender":"M","address":"945 Dunham Place","employer":"Playce","email":"elizapotter@playce.com","city":"Woodruff","state":"AK","_is_real":true} +{"index":{"_id":"713"}} +{"account_number":713,"balance":20054,"firstname":"Iris","lastname":"Mcguire","age":21,"gender":"F","address":"508 Benson Avenue","employer":"Duflex","email":"irismcguire@duflex.com","city":"Hillsboro","state":"MO","_is_real":true} +{"index":{"_id":"718"}} +{"account_number":718,"balance":13876,"firstname":"Hickman","lastname":"Dillard","age":22,"gender":"F","address":"132 Etna Street","employer":"Genmy","email":"hickmandillard@genmy.com","city":"Curtice","state":"NV","_is_real":true} +{"index":{"_id":"720"}} +{"account_number":720,"balance":31356,"firstname":"Ruth","lastname":"Vance","age":32,"gender":"F","address":"229 Adams Street","employer":"Zilidium","email":"ruthvance@zilidium.com","city":"Allison","state":"IA","_is_real":true} +{"index":{"_id":"725"}} +{"account_number":725,"balance":14677,"firstname":"Reeves","lastname":"Tillman","age":26,"gender":"M","address":"674 Ivan Court","employer":"Cemention","email":"reevestillman@cemention.com","city":"Navarre","state":"MA","_is_real":true} +{"index":{"_id":"732"}} +{"account_number":732,"balance":38445,"firstname":"Delia","lastname":"Cruz","age":37,"gender":"F","address":"870 Cheever Place","employer":"Multron","email":"deliacruz@multron.com","city":"Cresaptown","state":"NH","_is_real":true} +{"index":{"_id":"737"}} +{"account_number":737,"balance":40431,"firstname":"Sampson","lastname":"Yates","age":23,"gender":"F","address":"214 Cox Place","employer":"Signidyne","email":"sampsonyates@signidyne.com","city":"Brazos","state":"GA","_is_real":true} +{"index":{"_id":"744"}} +{"account_number":744,"balance":8690,"firstname":"Bernard","lastname":"Martinez","age":21,"gender":"M","address":"148 Dunne Place","employer":"Dragbot","email":"bernardmartinez@dragbot.com","city":"Moraida","state":"MN","_is_real":true} +{"index":{"_id":"749"}} +{"account_number":749,"balance":1249,"firstname":"Rush","lastname":"Boyle","age":36,"gender":"M","address":"310 Argyle Road","employer":"Sportan","email":"rushboyle@sportan.com","city":"Brady","state":"WA","_is_real":true} +{"index":{"_id":"751"}} +{"account_number":751,"balance":49252,"firstname":"Patrick","lastname":"Osborne","age":23,"gender":"M","address":"915 Prospect Avenue","employer":"Gynko","email":"patrickosborne@gynko.com","city":"Takilma","state":"MO","_is_real":true} +{"index":{"_id":"756"}} +{"account_number":756,"balance":40006,"firstname":"Jasmine","lastname":"Howell","age":32,"gender":"M","address":"605 Elliott Walk","employer":"Ecratic","email":"jasminehowell@ecratic.com","city":"Harrodsburg","state":"OH","_is_real":true} +{"index":{"_id":"763"}} +{"account_number":763,"balance":12091,"firstname":"Liz","lastname":"Bentley","age":22,"gender":"F","address":"933 Debevoise Avenue","employer":"Nipaz","email":"lizbentley@nipaz.com","city":"Glenville","state":"NJ","_is_real":true} +{"index":{"_id":"768"}} +{"account_number":768,"balance":2213,"firstname":"Sondra","lastname":"Soto","age":21,"gender":"M","address":"625 Colonial Road","employer":"Navir","email":"sondrasoto@navir.com","city":"Benson","state":"VA","_is_real":true} +{"index":{"_id":"770"}} +{"account_number":770,"balance":39505,"firstname":"Joann","lastname":"Crane","age":26,"gender":"M","address":"798 Farragut Place","employer":"Lingoage","email":"joanncrane@lingoage.com","city":"Kirk","state":"MA","_is_real":true} +{"index":{"_id":"775"}} +{"account_number":775,"balance":27943,"firstname":"Wilson","lastname":"Merritt","age":33,"gender":"F","address":"288 Thornton Street","employer":"Geeky","email":"wilsonmerritt@geeky.com","city":"Holtville","state":"HI","_is_real":true} +{"index":{"_id":"782"}} +{"account_number":782,"balance":3960,"firstname":"Maldonado","lastname":"Craig","age":36,"gender":"F","address":"345 Myrtle Avenue","employer":"Zilencio","email":"maldonadocraig@zilencio.com","city":"Yukon","state":"ID","_is_real":true} +{"index":{"_id":"787"}} +{"account_number":787,"balance":11876,"firstname":"Harper","lastname":"Wynn","age":21,"gender":"F","address":"139 Oceanic Avenue","employer":"Interfind","email":"harperwynn@interfind.com","city":"Gerber","state":"ND","_is_real":true} +{"index":{"_id":"794"}} +{"account_number":794,"balance":16491,"firstname":"Walker","lastname":"Charles","age":32,"gender":"M","address":"215 Kenilworth Place","employer":"Orbin","email":"walkercharles@orbin.com","city":"Rivers","state":"WI","_is_real":true} +{"index":{"_id":"799"}} +{"account_number":799,"balance":2889,"firstname":"Myra","lastname":"Guerra","age":28,"gender":"F","address":"625 Dahlgreen Place","employer":"Digigene","email":"myraguerra@digigene.com","city":"Draper","state":"CA","_is_real":true} +{"index":{"_id":"802"}} +{"account_number":802,"balance":19630,"firstname":"Gracie","lastname":"Foreman","age":40,"gender":"F","address":"219 Kent Avenue","employer":"Supportal","email":"gracieforeman@supportal.com","city":"Westboro","state":"NH","_is_real":true} +{"index":{"_id":"807"}} +{"account_number":807,"balance":29206,"firstname":"Hatfield","lastname":"Lowe","age":23,"gender":"M","address":"499 Adler Place","employer":"Lovepad","email":"hatfieldlowe@lovepad.com","city":"Wiscon","state":"DC","_is_real":true} +{"index":{"_id":"814"}} +{"account_number":814,"balance":9838,"firstname":"Morse","lastname":"Mcbride","age":26,"gender":"F","address":"776 Calyer Street","employer":"Inear","email":"morsemcbride@inear.com","city":"Kingstowne","state":"ND","_is_real":true} +{"index":{"_id":"819"}} +{"account_number":819,"balance":3971,"firstname":"Karyn","lastname":"Medina","age":24,"gender":"F","address":"417 Utica Avenue","employer":"Qnekt","email":"karynmedina@qnekt.com","city":"Kerby","state":"WY","_is_real":true} +{"index":{"_id":"821"}} +{"account_number":821,"balance":33271,"firstname":"Trisha","lastname":"Blankenship","age":22,"gender":"M","address":"329 Jamaica Avenue","employer":"Chorizon","email":"trishablankenship@chorizon.com","city":"Sexton","state":"VT","_is_real":true} +{"index":{"_id":"826"}} +{"account_number":826,"balance":11548,"firstname":"Summers","lastname":"Vinson","age":22,"gender":"F","address":"742 Irwin Street","employer":"Globoil","email":"summersvinson@globoil.com","city":"Callaghan","state":"WY","_is_real":true} +{"index":{"_id":"833"}} +{"account_number":833,"balance":46154,"firstname":"Woodward","lastname":"Hood","age":22,"gender":"M","address":"398 Atkins Avenue","employer":"Zedalis","email":"woodwardhood@zedalis.com","city":"Stonybrook","state":"NE","_is_real":true} +{"index":{"_id":"838"}} +{"account_number":838,"balance":24629,"firstname":"Latonya","lastname":"Blake","age":37,"gender":"F","address":"531 Milton Street","employer":"Rugstars","email":"latonyablake@rugstars.com","city":"Tedrow","state":"WA","_is_real":true} +{"index":{"_id":"840"}} +{"account_number":840,"balance":39615,"firstname":"Boone","lastname":"Gomez","age":38,"gender":"M","address":"256 Hampton Place","employer":"Geekular","email":"boonegomez@geekular.com","city":"Westerville","state":"HI","_is_real":true} +{"index":{"_id":"845"}} +{"account_number":845,"balance":35422,"firstname":"Tracy","lastname":"Vaughn","age":39,"gender":"M","address":"645 Rockaway Parkway","employer":"Andryx","email":"tracyvaughn@andryx.com","city":"Wilmington","state":"ME","_is_real":true} +{"index":{"_id":"852"}} +{"account_number":852,"balance":6041,"firstname":"Allen","lastname":"Hammond","age":26,"gender":"M","address":"793 Essex Street","employer":"Tersanki","email":"allenhammond@tersanki.com","city":"Osmond","state":"NC","_is_real":true} +{"index":{"_id":"857"}} +{"account_number":857,"balance":39678,"firstname":"Alyce","lastname":"Douglas","age":23,"gender":"M","address":"326 Robert Street","employer":"Earbang","email":"alycedouglas@earbang.com","city":"Thornport","state":"GA","_is_real":true} +{"index":{"_id":"864"}} +{"account_number":864,"balance":21804,"firstname":"Duffy","lastname":"Anthony","age":23,"gender":"M","address":"582 Cooke Court","employer":"Schoolio","email":"duffyanthony@schoolio.com","city":"Brenton","state":"CO","_is_real":true} +{"index":{"_id":"869"}} +{"account_number":869,"balance":43544,"firstname":"Corinne","lastname":"Robbins","age":25,"gender":"F","address":"732 Quentin Road","employer":"Orbaxter","email":"corinnerobbins@orbaxter.com","city":"Roy","state":"TN","_is_real":true} +{"index":{"_id":"871"}} +{"account_number":871,"balance":35854,"firstname":"Norma","lastname":"Burt","age":32,"gender":"M","address":"934 Cyrus Avenue","employer":"Magnafone","email":"normaburt@magnafone.com","city":"Eden","state":"TN","_is_real":true} +{"index":{"_id":"876"}} +{"account_number":876,"balance":48568,"firstname":"Brady","lastname":"Glover","age":21,"gender":"F","address":"565 Oceanview Avenue","employer":"Comvex","email":"bradyglover@comvex.com","city":"Noblestown","state":"ID","_is_real":true} +{"index":{"_id":"883"}} +{"account_number":883,"balance":33679,"firstname":"Austin","lastname":"Jefferson","age":34,"gender":"M","address":"846 Lincoln Avenue","employer":"Polarax","email":"austinjefferson@polarax.com","city":"Savannah","state":"CT","_is_real":true} +{"index":{"_id":"888"}} +{"account_number":888,"balance":22277,"firstname":"Myrna","lastname":"Herman","age":39,"gender":"F","address":"649 Harwood Place","employer":"Enthaze","email":"myrnaherman@enthaze.com","city":"Idamay","state":"AR","_is_real":true} +{"index":{"_id":"890"}} +{"account_number":890,"balance":31198,"firstname":"Alvarado","lastname":"Pate","age":25,"gender":"M","address":"269 Ashland Place","employer":"Ovolo","email":"alvaradopate@ovolo.com","city":"Volta","state":"MI","_is_real":true} +{"index":{"_id":"895"}} +{"account_number":895,"balance":7327,"firstname":"Lara","lastname":"Mcdaniel","age":36,"gender":"M","address":"854 Willow Place","employer":"Acusage","email":"laramcdaniel@acusage.com","city":"Imperial","state":"NC","_is_real":true} +{"index":{"_id":"903"}} +{"account_number":903,"balance":10238,"firstname":"Wade","lastname":"Page","age":35,"gender":"F","address":"685 Waldorf Court","employer":"Eplosion","email":"wadepage@eplosion.com","city":"Welda","state":"AL","_is_real":true} +{"index":{"_id":"908"}} +{"account_number":908,"balance":45975,"firstname":"Mosley","lastname":"Holloway","age":31,"gender":"M","address":"929 Eldert Lane","employer":"Anivet","email":"mosleyholloway@anivet.com","city":"Biehle","state":"MS","_is_real":true} +{"index":{"_id":"910"}} +{"account_number":910,"balance":36831,"firstname":"Esmeralda","lastname":"James","age":23,"gender":"F","address":"535 High Street","employer":"Terrasys","email":"esmeraldajames@terrasys.com","city":"Dubois","state":"IN","_is_real":true} +{"index":{"_id":"915"}} +{"account_number":915,"balance":19816,"firstname":"Farrell","lastname":"French","age":35,"gender":"F","address":"126 McKibbin Street","employer":"Techmania","email":"farrellfrench@techmania.com","city":"Wescosville","state":"AL","_is_real":true} +{"index":{"_id":"922"}} +{"account_number":922,"balance":39347,"firstname":"Irwin","lastname":"Pugh","age":32,"gender":"M","address":"463 Shale Street","employer":"Idego","email":"irwinpugh@idego.com","city":"Ivanhoe","state":"ID","_is_real":true} +{"index":{"_id":"927"}} +{"account_number":927,"balance":19976,"firstname":"Jeanette","lastname":"Acevedo","age":26,"gender":"M","address":"694 Polhemus Place","employer":"Halap","email":"jeanetteacevedo@halap.com","city":"Harrison","state":"MO","_is_real":true} +{"index":{"_id":"934"}} +{"account_number":934,"balance":43987,"firstname":"Freida","lastname":"Daniels","age":34,"gender":"M","address":"448 Cove Lane","employer":"Vurbo","email":"freidadaniels@vurbo.com","city":"Snelling","state":"NJ","_is_real":true} +{"index":{"_id":"939"}} +{"account_number":939,"balance":31228,"firstname":"Hodges","lastname":"Massey","age":37,"gender":"F","address":"431 Dahl Court","employer":"Kegular","email":"hodgesmassey@kegular.com","city":"Katonah","state":"MD","_is_real":true} +{"index":{"_id":"941"}} +{"account_number":941,"balance":38796,"firstname":"Kim","lastname":"Moss","age":28,"gender":"F","address":"105 Onderdonk Avenue","employer":"Digirang","email":"kimmoss@digirang.com","city":"Centerville","state":"TX","_is_real":true} +{"index":{"_id":"946"}} +{"account_number":946,"balance":42794,"firstname":"Ina","lastname":"Obrien","age":36,"gender":"M","address":"339 Rewe Street","employer":"Eclipsent","email":"inaobrien@eclipsent.com","city":"Soham","state":"RI","_is_real":true} +{"index":{"_id":"953"}} +{"account_number":953,"balance":1110,"firstname":"Baxter","lastname":"Black","age":27,"gender":"M","address":"720 Stillwell Avenue","employer":"Uplinx","email":"baxterblack@uplinx.com","city":"Drummond","state":"MN","_is_real":true} +{"index":{"_id":"958"}} +{"account_number":958,"balance":32849,"firstname":"Brown","lastname":"Wilkins","age":40,"gender":"M","address":"686 Delmonico Place","employer":"Medesign","email":"brownwilkins@medesign.com","city":"Shelby","state":"WY","_is_real":true} +{"index":{"_id":"960"}} +{"account_number":960,"balance":2905,"firstname":"Curry","lastname":"Vargas","age":40,"gender":"M","address":"242 Blake Avenue","employer":"Pearlesex","email":"curryvargas@pearlesex.com","city":"Henrietta","state":"NH","_is_real":true} +{"index":{"_id":"965"}} +{"account_number":965,"balance":21882,"firstname":"Patrica","lastname":"Melton","age":28,"gender":"M","address":"141 Rodney Street","employer":"Flexigen","email":"patricamelton@flexigen.com","city":"Klagetoh","state":"MD","_is_real":true} +{"index":{"_id":"972"}} +{"account_number":972,"balance":24719,"firstname":"Leona","lastname":"Christian","age":26,"gender":"F","address":"900 Woodpoint Road","employer":"Extrawear","email":"leonachristian@extrawear.com","city":"Roderfield","state":"MA","_is_real":true} +{"index":{"_id":"977"}} +{"account_number":977,"balance":6744,"firstname":"Rodgers","lastname":"Mccray","age":21,"gender":"F","address":"612 Duryea Place","employer":"Papricut","email":"rodgersmccray@papricut.com","city":"Marenisco","state":"MD","_is_real":true} +{"index":{"_id":"984"}} +{"account_number":984,"balance":1904,"firstname":"Viola","lastname":"Crawford","age":35,"gender":"F","address":"354 Linwood Street","employer":"Ginkle","email":"violacrawford@ginkle.com","city":"Witmer","state":"AR","_is_real":true} +{"index":{"_id":"989"}} +{"account_number":989,"balance":48622,"firstname":"Franklin","lastname":"Frank","age":38,"gender":"M","address":"270 Carlton Avenue","employer":"Shopabout","email":"franklinfrank@shopabout.com","city":"Guthrie","state":"NC","_is_real":true} +{"index":{"_id":"991"}} +{"account_number":991,"balance":4239,"firstname":"Connie","lastname":"Berry","age":28,"gender":"F","address":"647 Gardner Avenue","employer":"Flumbo","email":"connieberry@flumbo.com","city":"Frierson","state":"MO","_is_real":true} +{"index":{"_id":"996"}} +{"account_number":996,"balance":17541,"firstname":"Andrews","lastname":"Herrera","age":30,"gender":"F","address":"570 Vandam Street","employer":"Klugger","email":"andrewsherrera@klugger.com","city":"Whitehaven","state":"MN","_is_real":true} +{"index":{"_id":"0"}} +{"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO","_is_real":true} +{"index":{"_id":"5"}} +{"account_number":5,"balance":29342,"firstname":"Leola","lastname":"Stewart","age":30,"gender":"F","address":"311 Elm Place","employer":"Diginetic","email":"leolastewart@diginetic.com","city":"Fairview","state":"NJ","_is_real":true} +{"index":{"_id":"12"}} +{"account_number":12,"balance":37055,"firstname":"Stafford","lastname":"Brock","age":20,"gender":"F","address":"296 Wythe Avenue","employer":"Uncorp","email":"staffordbrock@uncorp.com","city":"Bend","state":"AL","_is_real":true} +{"index":{"_id":"17"}} +{"account_number":17,"balance":7831,"firstname":"Bessie","lastname":"Orr","age":31,"gender":"F","address":"239 Hinsdale Street","employer":"Skyplex","email":"bessieorr@skyplex.com","city":"Graball","state":"FL","_is_real":true} +{"index":{"_id":"24"}} +{"account_number":24,"balance":44182,"firstname":"Wood","lastname":"Dale","age":39,"gender":"M","address":"582 Gelston Avenue","employer":"Besto","email":"wooddale@besto.com","city":"Juntura","state":"MI","_is_real":true} +{"index":{"_id":"29"}} +{"account_number":29,"balance":27323,"firstname":"Leah","lastname":"Santiago","age":33,"gender":"M","address":"193 Schenck Avenue","employer":"Isologix","email":"leahsantiago@isologix.com","city":"Gerton","state":"ND","_is_real":true} +{"index":{"_id":"31"}} +{"account_number":31,"balance":30443,"firstname":"Kristen","lastname":"Santana","age":22,"gender":"F","address":"130 Middagh Street","employer":"Dogspa","email":"kristensantana@dogspa.com","city":"Vale","state":"MA","_is_real":true} +{"index":{"_id":"36"}} +{"account_number":36,"balance":15902,"firstname":"Alexandra","lastname":"Nguyen","age":39,"gender":"F","address":"389 Elizabeth Place","employer":"Bittor","email":"alexandranguyen@bittor.com","city":"Hemlock","state":"KY","_is_real":true} +{"index":{"_id":"43"}} +{"account_number":43,"balance":33474,"firstname":"Ryan","lastname":"Howe","age":25,"gender":"M","address":"660 Huntington Street","employer":"Microluxe","email":"ryanhowe@microluxe.com","city":"Clara","state":"CT","_is_real":true} +{"index":{"_id":"48"}} +{"account_number":48,"balance":40608,"firstname":"Peck","lastname":"Downs","age":39,"gender":"F","address":"594 Dwight Street","employer":"Ramjob","email":"peckdowns@ramjob.com","city":"Coloma","state":"WA","_is_real":true} +{"index":{"_id":"50"}} +{"account_number":50,"balance":43695,"firstname":"Sheena","lastname":"Kirkland","age":33,"gender":"M","address":"598 Bank Street","employer":"Zerbina","email":"sheenakirkland@zerbina.com","city":"Walland","state":"IN","_is_real":true} +{"index":{"_id":"55"}} +{"account_number":55,"balance":22020,"firstname":"Shelia","lastname":"Puckett","age":33,"gender":"M","address":"265 Royce Place","employer":"Izzby","email":"sheliapuckett@izzby.com","city":"Slovan","state":"HI","_is_real":true} +{"index":{"_id":"62"}} +{"account_number":62,"balance":43065,"firstname":"Lester","lastname":"Stanton","age":37,"gender":"M","address":"969 Doughty Street","employer":"Geekko","email":"lesterstanton@geekko.com","city":"Itmann","state":"DC","_is_real":true} +{"index":{"_id":"67"}} +{"account_number":67,"balance":39430,"firstname":"Isabelle","lastname":"Spence","age":39,"gender":"M","address":"718 Troy Avenue","employer":"Geeketron","email":"isabellespence@geeketron.com","city":"Camptown","state":"WA","_is_real":true} +{"index":{"_id":"74"}} +{"account_number":74,"balance":47167,"firstname":"Lauri","lastname":"Saunders","age":38,"gender":"F","address":"768 Lynch Street","employer":"Securia","email":"laurisaunders@securia.com","city":"Caroline","state":"TN","_is_real":true} +{"index":{"_id":"79"}} +{"account_number":79,"balance":28185,"firstname":"Booker","lastname":"Lowery","age":29,"gender":"M","address":"817 Campus Road","employer":"Sensate","email":"bookerlowery@sensate.com","city":"Carlos","state":"MT","_is_real":true} +{"index":{"_id":"81"}} +{"account_number":81,"balance":46568,"firstname":"Dennis","lastname":"Gilbert","age":40,"gender":"M","address":"619 Minna Street","employer":"Melbacor","email":"dennisgilbert@melbacor.com","city":"Kersey","state":"ND","_is_real":true} +{"index":{"_id":"86"}} +{"account_number":86,"balance":15428,"firstname":"Walton","lastname":"Butler","age":36,"gender":"M","address":"999 Schenck Street","employer":"Unisure","email":"waltonbutler@unisure.com","city":"Bentonville","state":"IL","_is_real":true} +{"index":{"_id":"93"}} +{"account_number":93,"balance":17728,"firstname":"Jeri","lastname":"Booth","age":31,"gender":"M","address":"322 Roosevelt Court","employer":"Geekology","email":"jeribooth@geekology.com","city":"Leming","state":"ND","_is_real":true} +{"index":{"_id":"98"}} +{"account_number":98,"balance":15085,"firstname":"Cora","lastname":"Barrett","age":24,"gender":"F","address":"555 Neptune Court","employer":"Kiosk","email":"corabarrett@kiosk.com","city":"Independence","state":"MN","_is_real":true} +{"index":{"_id":"101"}} +{"account_number":101,"balance":43400,"firstname":"Cecelia","lastname":"Grimes","age":31,"gender":"M","address":"972 Lincoln Place","employer":"Ecosys","email":"ceceliagrimes@ecosys.com","city":"Manchester","state":"AR","_is_real":true} +{"index":{"_id":"106"}} +{"account_number":106,"balance":8212,"firstname":"Josefina","lastname":"Wagner","age":36,"gender":"M","address":"418 Estate Road","employer":"Kyaguru","email":"josefinawagner@kyaguru.com","city":"Darbydale","state":"FL","_is_real":true} +{"index":{"_id":"113"}} +{"account_number":113,"balance":41652,"firstname":"Burt","lastname":"Moses","age":27,"gender":"M","address":"633 Berry Street","employer":"Uni","email":"burtmoses@uni.com","city":"Russellville","state":"CT","_is_real":true} +{"index":{"_id":"118"}} +{"account_number":118,"balance":2223,"firstname":"Ballard","lastname":"Vasquez","age":33,"gender":"F","address":"101 Bush Street","employer":"Intergeek","email":"ballardvasquez@intergeek.com","city":"Century","state":"MN","_is_real":true} +{"index":{"_id":"120"}} +{"account_number":120,"balance":38565,"firstname":"Browning","lastname":"Rodriquez","age":33,"gender":"M","address":"910 Moore Street","employer":"Opportech","email":"browningrodriquez@opportech.com","city":"Cutter","state":"ND","_is_real":true} +{"index":{"_id":"125"}} +{"account_number":125,"balance":5396,"firstname":"Tanisha","lastname":"Dixon","age":30,"gender":"M","address":"482 Hancock Street","employer":"Junipoor","email":"tanishadixon@junipoor.com","city":"Wauhillau","state":"IA","_is_real":true} +{"index":{"_id":"132"}} +{"account_number":132,"balance":37707,"firstname":"Horton","lastname":"Romero","age":35,"gender":"M","address":"427 Rutherford Place","employer":"Affluex","email":"hortonromero@affluex.com","city":"Hall","state":"AK","_is_real":true} +{"index":{"_id":"137"}} +{"account_number":137,"balance":3596,"firstname":"Frost","lastname":"Freeman","age":29,"gender":"F","address":"191 Dennett Place","employer":"Beadzza","email":"frostfreeman@beadzza.com","city":"Sabillasville","state":"HI","_is_real":true} +{"index":{"_id":"144"}} +{"account_number":144,"balance":43257,"firstname":"Evans","lastname":"Dyer","age":30,"gender":"F","address":"912 Post Court","employer":"Magmina","email":"evansdyer@magmina.com","city":"Gordon","state":"HI","_is_real":true} +{"index":{"_id":"149"}} +{"account_number":149,"balance":22994,"firstname":"Megan","lastname":"Gonzales","age":21,"gender":"M","address":"836 Tampa Court","employer":"Andershun","email":"megangonzales@andershun.com","city":"Rockhill","state":"AL","_is_real":true} +{"index":{"_id":"151"}} +{"account_number":151,"balance":34473,"firstname":"Kent","lastname":"Joyner","age":20,"gender":"F","address":"799 Truxton Street","employer":"Kozgene","email":"kentjoyner@kozgene.com","city":"Allamuchy","state":"DC","_is_real":true} +{"index":{"_id":"156"}} +{"account_number":156,"balance":40185,"firstname":"Sloan","lastname":"Pennington","age":24,"gender":"F","address":"573 Opal Court","employer":"Hopeli","email":"sloanpennington@hopeli.com","city":"Evergreen","state":"CT","_is_real":true} +{"index":{"_id":"163"}} +{"account_number":163,"balance":43075,"firstname":"Wilda","lastname":"Norman","age":33,"gender":"F","address":"173 Beadel Street","employer":"Kog","email":"wildanorman@kog.com","city":"Bodega","state":"ME","_is_real":true} +{"index":{"_id":"168"}} +{"account_number":168,"balance":49568,"firstname":"Carissa","lastname":"Simon","age":20,"gender":"M","address":"975 Flatbush Avenue","employer":"Zillacom","email":"carissasimon@zillacom.com","city":"Neibert","state":"IL","_is_real":true} +{"index":{"_id":"170"}} +{"account_number":170,"balance":6025,"firstname":"Mann","lastname":"Madden","age":36,"gender":"F","address":"161 Radde Place","employer":"Farmex","email":"mannmadden@farmex.com","city":"Thermal","state":"LA","_is_real":true} +{"index":{"_id":"175"}} +{"account_number":175,"balance":16213,"firstname":"Montoya","lastname":"Donaldson","age":28,"gender":"F","address":"481 Morton Street","employer":"Envire","email":"montoyadonaldson@envire.com","city":"Delco","state":"MA","_is_real":true} +{"index":{"_id":"182"}} +{"account_number":182,"balance":7803,"firstname":"Manuela","lastname":"Dillon","age":21,"gender":"M","address":"742 Garnet Street","employer":"Moreganic","email":"manueladillon@moreganic.com","city":"Ilchester","state":"TX","_is_real":true} +{"index":{"_id":"187"}} +{"account_number":187,"balance":26581,"firstname":"Autumn","lastname":"Hodges","age":35,"gender":"M","address":"757 Granite Street","employer":"Ezentia","email":"autumnhodges@ezentia.com","city":"Martinsville","state":"KY","_is_real":true} +{"index":{"_id":"194"}} +{"account_number":194,"balance":16311,"firstname":"Beck","lastname":"Rosario","age":39,"gender":"M","address":"721 Cambridge Place","employer":"Zoid","email":"beckrosario@zoid.com","city":"Efland","state":"ID","_is_real":true} +{"index":{"_id":"199"}} +{"account_number":199,"balance":18086,"firstname":"Branch","lastname":"Love","age":26,"gender":"M","address":"458 Commercial Street","employer":"Frolix","email":"branchlove@frolix.com","city":"Caspar","state":"NC","_is_real":true} +{"index":{"_id":"202"}} +{"account_number":202,"balance":26466,"firstname":"Medina","lastname":"Brown","age":31,"gender":"F","address":"519 Sunnyside Court","employer":"Bleendot","email":"medinabrown@bleendot.com","city":"Winfred","state":"MI","_is_real":true} +{"index":{"_id":"207"}} +{"account_number":207,"balance":45535,"firstname":"Evelyn","lastname":"Lara","age":35,"gender":"F","address":"636 Chestnut Street","employer":"Ultrasure","email":"evelynlara@ultrasure.com","city":"Logan","state":"MI","_is_real":true} +{"index":{"_id":"214"}} +{"account_number":214,"balance":24418,"firstname":"Luann","lastname":"Faulkner","age":37,"gender":"F","address":"697 Hazel Court","employer":"Zolar","email":"luannfaulkner@zolar.com","city":"Ticonderoga","state":"TX","_is_real":true} +{"index":{"_id":"219"}} +{"account_number":219,"balance":17127,"firstname":"Edwards","lastname":"Hurley","age":25,"gender":"M","address":"834 Stockholm Street","employer":"Austech","email":"edwardshurley@austech.com","city":"Bayview","state":"NV","_is_real":true} +{"index":{"_id":"221"}} +{"account_number":221,"balance":15803,"firstname":"Benjamin","lastname":"Barrera","age":34,"gender":"M","address":"568 Main Street","employer":"Zaphire","email":"benjaminbarrera@zaphire.com","city":"Germanton","state":"WY","_is_real":true} +{"index":{"_id":"226"}} +{"account_number":226,"balance":37720,"firstname":"Wilkins","lastname":"Brady","age":40,"gender":"F","address":"486 Baltic Street","employer":"Dogtown","email":"wilkinsbrady@dogtown.com","city":"Condon","state":"MT","_is_real":true} +{"index":{"_id":"233"}} +{"account_number":233,"balance":23020,"firstname":"Washington","lastname":"Walsh","age":27,"gender":"M","address":"366 Church Avenue","employer":"Candecor","email":"washingtonwalsh@candecor.com","city":"Westphalia","state":"MA","_is_real":true} +{"index":{"_id":"238"}} +{"account_number":238,"balance":21287,"firstname":"Constance","lastname":"Wong","age":28,"gender":"M","address":"496 Brown Street","employer":"Grainspot","email":"constancewong@grainspot.com","city":"Cecilia","state":"IN","_is_real":true} +{"index":{"_id":"240"}} +{"account_number":240,"balance":49741,"firstname":"Oconnor","lastname":"Clay","age":35,"gender":"F","address":"659 Highland Boulevard","employer":"Franscene","email":"oconnorclay@franscene.com","city":"Kilbourne","state":"NH","_is_real":true} +{"index":{"_id":"245"}} +{"account_number":245,"balance":22026,"firstname":"Fran","lastname":"Bolton","age":28,"gender":"F","address":"147 Jerome Street","employer":"Solaren","email":"franbolton@solaren.com","city":"Nash","state":"RI","_is_real":true} +{"index":{"_id":"252"}} +{"account_number":252,"balance":18831,"firstname":"Elvia","lastname":"Poole","age":22,"gender":"F","address":"836 Delevan Street","employer":"Velity","email":"elviapoole@velity.com","city":"Groveville","state":"MI","_is_real":true} +{"index":{"_id":"257"}} +{"account_number":257,"balance":5318,"firstname":"Olive","lastname":"Oneil","age":35,"gender":"F","address":"457 Decatur Street","employer":"Helixo","email":"oliveoneil@helixo.com","city":"Chicopee","state":"MI","_is_real":true} +{"index":{"_id":"264"}} +{"account_number":264,"balance":22084,"firstname":"Samantha","lastname":"Ferrell","age":35,"gender":"F","address":"488 Fulton Street","employer":"Flum","email":"samanthaferrell@flum.com","city":"Brandywine","state":"MT","_is_real":true} +{"index":{"_id":"269"}} +{"account_number":269,"balance":43317,"firstname":"Crosby","lastname":"Figueroa","age":34,"gender":"M","address":"910 Aurelia Court","employer":"Pyramia","email":"crosbyfigueroa@pyramia.com","city":"Leyner","state":"OH","_is_real":true} +{"index":{"_id":"271"}} +{"account_number":271,"balance":11864,"firstname":"Holt","lastname":"Walter","age":30,"gender":"F","address":"645 Poplar Avenue","employer":"Grupoli","email":"holtwalter@grupoli.com","city":"Mansfield","state":"OR","_is_real":true} +{"index":{"_id":"276"}} +{"account_number":276,"balance":11606,"firstname":"Pittman","lastname":"Mathis","age":23,"gender":"F","address":"567 Charles Place","employer":"Zuvy","email":"pittmanmathis@zuvy.com","city":"Roeville","state":"KY","_is_real":true} +{"index":{"_id":"283"}} +{"account_number":283,"balance":24070,"firstname":"Fuentes","lastname":"Foley","age":30,"gender":"M","address":"729 Walker Court","employer":"Knowlysis","email":"fuentesfoley@knowlysis.com","city":"Tryon","state":"TN","_is_real":true} +{"index":{"_id":"288"}} +{"account_number":288,"balance":27243,"firstname":"Wong","lastname":"Stone","age":39,"gender":"F","address":"440 Willoughby Street","employer":"Zentix","email":"wongstone@zentix.com","city":"Wheatfields","state":"DC","_is_real":true} +{"index":{"_id":"290"}} +{"account_number":290,"balance":26103,"firstname":"Neva","lastname":"Burgess","age":37,"gender":"F","address":"985 Wyona Street","employer":"Slofast","email":"nevaburgess@slofast.com","city":"Cawood","state":"DC","_is_real":true} +{"index":{"_id":"295"}} +{"account_number":295,"balance":37358,"firstname":"Howe","lastname":"Nash","age":20,"gender":"M","address":"833 Union Avenue","employer":"Aquacine","email":"howenash@aquacine.com","city":"Indio","state":"MN","_is_real":true} +{"index":{"_id":"303"}} +{"account_number":303,"balance":21976,"firstname":"Huffman","lastname":"Green","age":24,"gender":"F","address":"455 Colby Court","employer":"Comtest","email":"huffmangreen@comtest.com","city":"Weeksville","state":"UT","_is_real":true} +{"index":{"_id":"308"}} +{"account_number":308,"balance":33989,"firstname":"Glass","lastname":"Schroeder","age":25,"gender":"F","address":"670 Veterans Avenue","employer":"Realmo","email":"glassschroeder@realmo.com","city":"Gratton","state":"NY","_is_real":true} +{"index":{"_id":"310"}} +{"account_number":310,"balance":23049,"firstname":"Shannon","lastname":"Morton","age":39,"gender":"F","address":"412 Pleasant Place","employer":"Ovation","email":"shannonmorton@ovation.com","city":"Edgar","state":"AZ","_is_real":true} +{"index":{"_id":"315"}} +{"account_number":315,"balance":1314,"firstname":"Clare","lastname":"Morrow","age":33,"gender":"F","address":"728 Madeline Court","employer":"Gaptec","email":"claremorrow@gaptec.com","city":"Mapletown","state":"PA","_is_real":true} +{"index":{"_id":"322"}} +{"account_number":322,"balance":6303,"firstname":"Gilliam","lastname":"Horne","age":27,"gender":"M","address":"414 Florence Avenue","employer":"Shepard","email":"gilliamhorne@shepard.com","city":"Winesburg","state":"WY","_is_real":true} +{"index":{"_id":"327"}} +{"account_number":327,"balance":29294,"firstname":"Nell","lastname":"Contreras","age":27,"gender":"M","address":"694 Gold Street","employer":"Momentia","email":"nellcontreras@momentia.com","city":"Cumminsville","state":"AL","_is_real":true} +{"index":{"_id":"334"}} +{"account_number":334,"balance":9178,"firstname":"Cross","lastname":"Floyd","age":21,"gender":"F","address":"815 Herkimer Court","employer":"Maroptic","email":"crossfloyd@maroptic.com","city":"Kraemer","state":"AK","_is_real":true} +{"index":{"_id":"339"}} +{"account_number":339,"balance":3992,"firstname":"Franco","lastname":"Welch","age":38,"gender":"F","address":"776 Brightwater Court","employer":"Earthplex","email":"francowelch@earthplex.com","city":"Naomi","state":"ME","_is_real":true} +{"index":{"_id":"341"}} +{"account_number":341,"balance":44367,"firstname":"Alberta","lastname":"Bradford","age":30,"gender":"F","address":"670 Grant Avenue","employer":"Bugsall","email":"albertabradford@bugsall.com","city":"Romeville","state":"MT","_is_real":true} +{"index":{"_id":"346"}} +{"account_number":346,"balance":26594,"firstname":"Shelby","lastname":"Sanchez","age":36,"gender":"F","address":"257 Fillmore Avenue","employer":"Geekus","email":"shelbysanchez@geekus.com","city":"Seymour","state":"CO","_is_real":true} +{"index":{"_id":"353"}} +{"account_number":353,"balance":45182,"firstname":"Rivera","lastname":"Sherman","age":37,"gender":"M","address":"603 Garden Place","employer":"Bovis","email":"riverasherman@bovis.com","city":"Otranto","state":"CA","_is_real":true} +{"index":{"_id":"358"}} +{"account_number":358,"balance":44043,"firstname":"Hale","lastname":"Baldwin","age":40,"gender":"F","address":"845 Menahan Street","employer":"Kidgrease","email":"halebaldwin@kidgrease.com","city":"Day","state":"AK","_is_real":true} +{"index":{"_id":"360"}} +{"account_number":360,"balance":26651,"firstname":"Ward","lastname":"Hicks","age":34,"gender":"F","address":"592 Brighton Court","employer":"Biotica","email":"wardhicks@biotica.com","city":"Kanauga","state":"VT","_is_real":true} +{"index":{"_id":"365"}} +{"account_number":365,"balance":3176,"firstname":"Sanders","lastname":"Holder","age":31,"gender":"F","address":"453 Cypress Court","employer":"Geekola","email":"sandersholder@geekola.com","city":"Staples","state":"TN","_is_real":true} +{"index":{"_id":"372"}} +{"account_number":372,"balance":28566,"firstname":"Alba","lastname":"Forbes","age":24,"gender":"M","address":"814 Meserole Avenue","employer":"Isostream","email":"albaforbes@isostream.com","city":"Clarence","state":"OR","_is_real":true} +{"index":{"_id":"377"}} +{"account_number":377,"balance":5374,"firstname":"Margo","lastname":"Gay","age":34,"gender":"F","address":"613 Chase Court","employer":"Rotodyne","email":"margogay@rotodyne.com","city":"Waumandee","state":"KS","_is_real":true} +{"index":{"_id":"384"}} +{"account_number":384,"balance":48758,"firstname":"Sallie","lastname":"Houston","age":31,"gender":"F","address":"836 Polar Street","employer":"Squish","email":"salliehouston@squish.com","city":"Morningside","state":"NC","_is_real":true} +{"index":{"_id":"389"}} +{"account_number":389,"balance":8839,"firstname":"York","lastname":"Cummings","age":27,"gender":"M","address":"778 Centre Street","employer":"Insurity","email":"yorkcummings@insurity.com","city":"Freeburn","state":"RI","_is_real":true} +{"index":{"_id":"391"}} +{"account_number":391,"balance":14733,"firstname":"Holman","lastname":"Jordan","age":30,"gender":"M","address":"391 Forrest Street","employer":"Maineland","email":"holmanjordan@maineland.com","city":"Cade","state":"CT","_is_real":true} +{"index":{"_id":"396"}} +{"account_number":396,"balance":14613,"firstname":"Marsha","lastname":"Elliott","age":38,"gender":"F","address":"297 Liberty Avenue","employer":"Orbiflex","email":"marshaelliott@orbiflex.com","city":"Windsor","state":"TX","_is_real":true} +{"index":{"_id":"404"}} +{"account_number":404,"balance":34978,"firstname":"Massey","lastname":"Becker","age":26,"gender":"F","address":"930 Pitkin Avenue","employer":"Genekom","email":"masseybecker@genekom.com","city":"Blairstown","state":"OR","_is_real":true} +{"index":{"_id":"409"}} +{"account_number":409,"balance":36960,"firstname":"Maura","lastname":"Glenn","age":31,"gender":"M","address":"183 Poly Place","employer":"Viagreat","email":"mauraglenn@viagreat.com","city":"Foscoe","state":"DE","_is_real":true} +{"index":{"_id":"411"}} +{"account_number":411,"balance":1172,"firstname":"Guzman","lastname":"Whitfield","age":22,"gender":"M","address":"181 Perry Terrace","employer":"Springbee","email":"guzmanwhitfield@springbee.com","city":"Balm","state":"IN","_is_real":true} +{"index":{"_id":"416"}} +{"account_number":416,"balance":27169,"firstname":"Hunt","lastname":"Schwartz","age":28,"gender":"F","address":"461 Havens Place","employer":"Danja","email":"huntschwartz@danja.com","city":"Grenelefe","state":"NV","_is_real":true} +{"index":{"_id":"423"}} +{"account_number":423,"balance":38852,"firstname":"Hines","lastname":"Underwood","age":21,"gender":"F","address":"284 Louise Terrace","employer":"Namegen","email":"hinesunderwood@namegen.com","city":"Downsville","state":"CO","_is_real":true} +{"index":{"_id":"428"}} +{"account_number":428,"balance":13925,"firstname":"Stephens","lastname":"Cain","age":20,"gender":"F","address":"189 Summit Street","employer":"Rocklogic","email":"stephenscain@rocklogic.com","city":"Bourg","state":"HI","_is_real":true} +{"index":{"_id":"430"}} +{"account_number":430,"balance":15251,"firstname":"Alejandra","lastname":"Chavez","age":34,"gender":"M","address":"651 Butler Place","employer":"Gology","email":"alejandrachavez@gology.com","city":"Allensworth","state":"VT","_is_real":true} +{"index":{"_id":"435"}} +{"account_number":435,"balance":14654,"firstname":"Sue","lastname":"Lopez","age":22,"gender":"F","address":"632 Stone Avenue","employer":"Emergent","email":"suelopez@emergent.com","city":"Waterford","state":"TN","_is_real":true} +{"index":{"_id":"442"}} +{"account_number":442,"balance":36211,"firstname":"Lawanda","lastname":"Leon","age":27,"gender":"F","address":"126 Canal Avenue","employer":"Xixan","email":"lawandaleon@xixan.com","city":"Berwind","state":"TN","_is_real":true} +{"index":{"_id":"447"}} +{"account_number":447,"balance":11402,"firstname":"Lucia","lastname":"Livingston","age":35,"gender":"M","address":"773 Lake Avenue","employer":"Soprano","email":"lucialivingston@soprano.com","city":"Edgewater","state":"TN","_is_real":true} +{"index":{"_id":"454"}} +{"account_number":454,"balance":31687,"firstname":"Alicia","lastname":"Rollins","age":22,"gender":"F","address":"483 Verona Place","employer":"Boilcat","email":"aliciarollins@boilcat.com","city":"Lutsen","state":"MD","_is_real":true} +{"index":{"_id":"459"}} +{"account_number":459,"balance":18869,"firstname":"Pamela","lastname":"Henry","age":20,"gender":"F","address":"361 Locust Avenue","employer":"Imageflow","email":"pamelahenry@imageflow.com","city":"Greenfields","state":"OH","_is_real":true} +{"index":{"_id":"461"}} +{"account_number":461,"balance":38807,"firstname":"Mcbride","lastname":"Padilla","age":34,"gender":"F","address":"550 Borinquen Pl","employer":"Zepitope","email":"mcbridepadilla@zepitope.com","city":"Emory","state":"AZ","_is_real":true} +{"index":{"_id":"466"}} +{"account_number":466,"balance":25109,"firstname":"Marcie","lastname":"Mcmillan","age":30,"gender":"F","address":"947 Gain Court","employer":"Entroflex","email":"marciemcmillan@entroflex.com","city":"Ronco","state":"ND","_is_real":true} +{"index":{"_id":"473"}} +{"account_number":473,"balance":5391,"firstname":"Susan","lastname":"Luna","age":25,"gender":"F","address":"521 Bogart Street","employer":"Zaya","email":"susanluna@zaya.com","city":"Grazierville","state":"MI","_is_real":true} +{"index":{"_id":"478"}} +{"account_number":478,"balance":28044,"firstname":"Dana","lastname":"Decker","age":35,"gender":"M","address":"627 Dobbin Street","employer":"Acrodance","email":"danadecker@acrodance.com","city":"Sharon","state":"MN","_is_real":true} +{"index":{"_id":"480"}} +{"account_number":480,"balance":40807,"firstname":"Anastasia","lastname":"Parker","age":24,"gender":"M","address":"650 Folsom Place","employer":"Zilladyne","email":"anastasiaparker@zilladyne.com","city":"Oberlin","state":"WY","_is_real":true} +{"index":{"_id":"485"}} +{"account_number":485,"balance":44235,"firstname":"Albert","lastname":"Roberts","age":40,"gender":"M","address":"385 Harman Street","employer":"Stralum","email":"albertroberts@stralum.com","city":"Watrous","state":"NM","_is_real":true} +{"index":{"_id":"492"}} +{"account_number":492,"balance":31055,"firstname":"Burnett","lastname":"Briggs","age":35,"gender":"M","address":"987 Cass Place","employer":"Pharmex","email":"burnettbriggs@pharmex.com","city":"Cornfields","state":"TX","_is_real":true} +{"index":{"_id":"497"}} +{"account_number":497,"balance":13493,"firstname":"Doyle","lastname":"Jenkins","age":30,"gender":"M","address":"205 Nevins Street","employer":"Unia","email":"doylejenkins@unia.com","city":"Nicut","state":"DC","_is_real":true} +{"index":{"_id":"500"}} +{"account_number":500,"balance":39143,"firstname":"Pope","lastname":"Keith","age":28,"gender":"F","address":"537 Fane Court","employer":"Zboo","email":"popekeith@zboo.com","city":"Courtland","state":"AL","_is_real":true} +{"index":{"_id":"505"}} +{"account_number":505,"balance":45493,"firstname":"Shelley","lastname":"Webb","age":29,"gender":"M","address":"873 Crawford Avenue","employer":"Quadeebo","email":"shelleywebb@quadeebo.com","city":"Topanga","state":"IL","_is_real":true} +{"index":{"_id":"512"}} +{"account_number":512,"balance":47432,"firstname":"Alisha","lastname":"Morales","age":29,"gender":"M","address":"623 Batchelder Street","employer":"Terragen","email":"alishamorales@terragen.com","city":"Gilmore","state":"VA","_is_real":true} +{"index":{"_id":"517"}} +{"account_number":517,"balance":3022,"firstname":"Allyson","lastname":"Walls","age":38,"gender":"F","address":"334 Coffey Street","employer":"Gorganic","email":"allysonwalls@gorganic.com","city":"Dahlen","state":"GA","_is_real":true} +{"index":{"_id":"524"}} +{"account_number":524,"balance":49334,"firstname":"Salas","lastname":"Farley","age":30,"gender":"F","address":"499 Trucklemans Lane","employer":"Xumonk","email":"salasfarley@xumonk.com","city":"Noxen","state":"AL","_is_real":true} +{"index":{"_id":"529"}} +{"account_number":529,"balance":21788,"firstname":"Deann","lastname":"Fisher","age":23,"gender":"F","address":"511 Buffalo Avenue","employer":"Twiist","email":"deannfisher@twiist.com","city":"Templeton","state":"WA","_is_real":true} +{"index":{"_id":"531"}} +{"account_number":531,"balance":39770,"firstname":"Janet","lastname":"Pena","age":38,"gender":"M","address":"645 Livonia Avenue","employer":"Corecom","email":"janetpena@corecom.com","city":"Garberville","state":"OK","_is_real":true} +{"index":{"_id":"536"}} +{"account_number":536,"balance":6255,"firstname":"Emma","lastname":"Adkins","age":33,"gender":"F","address":"971 Calder Place","employer":"Ontagene","email":"emmaadkins@ontagene.com","city":"Ruckersville","state":"GA","_is_real":true} +{"index":{"_id":"543"}} +{"account_number":543,"balance":48022,"firstname":"Marina","lastname":"Rasmussen","age":31,"gender":"M","address":"446 Love Lane","employer":"Crustatia","email":"marinarasmussen@crustatia.com","city":"Statenville","state":"MD","_is_real":true} +{"index":{"_id":"548"}} +{"account_number":548,"balance":36930,"firstname":"Sandra","lastname":"Andrews","age":37,"gender":"M","address":"973 Prospect Street","employer":"Datagene","email":"sandraandrews@datagene.com","city":"Inkerman","state":"MO","_is_real":true} +{"index":{"_id":"550"}} +{"account_number":550,"balance":32238,"firstname":"Walsh","lastname":"Goodwin","age":22,"gender":"M","address":"953 Canda Avenue","employer":"Proflex","email":"walshgoodwin@proflex.com","city":"Ypsilanti","state":"MT","_is_real":true} +{"index":{"_id":"555"}} +{"account_number":555,"balance":10750,"firstname":"Fannie","lastname":"Slater","age":31,"gender":"M","address":"457 Tech Place","employer":"Kineticut","email":"fannieslater@kineticut.com","city":"Basye","state":"MO","_is_real":true} +{"index":{"_id":"562"}} +{"account_number":562,"balance":10737,"firstname":"Sarah","lastname":"Strong","age":39,"gender":"F","address":"177 Pioneer Street","employer":"Megall","email":"sarahstrong@megall.com","city":"Ladera","state":"WY","_is_real":true} +{"index":{"_id":"567"}} +{"account_number":567,"balance":6507,"firstname":"Diana","lastname":"Dominguez","age":40,"gender":"M","address":"419 Albany Avenue","employer":"Ohmnet","email":"dianadominguez@ohmnet.com","city":"Wildwood","state":"TX","_is_real":true} +{"index":{"_id":"574"}} +{"account_number":574,"balance":32954,"firstname":"Andrea","lastname":"Mosley","age":24,"gender":"M","address":"368 Throop Avenue","employer":"Musix","email":"andreamosley@musix.com","city":"Blende","state":"DC","_is_real":true} +{"index":{"_id":"579"}} +{"account_number":579,"balance":12044,"firstname":"Banks","lastname":"Sawyer","age":36,"gender":"M","address":"652 Doone Court","employer":"Rooforia","email":"bankssawyer@rooforia.com","city":"Foxworth","state":"ND","_is_real":true} +{"index":{"_id":"581"}} +{"account_number":581,"balance":16525,"firstname":"Fuller","lastname":"Mcintyre","age":32,"gender":"M","address":"169 Bergen Place","employer":"Applideck","email":"fullermcintyre@applideck.com","city":"Kenvil","state":"NY","_is_real":true} +{"index":{"_id":"586"}} +{"account_number":586,"balance":13644,"firstname":"Love","lastname":"Velasquez","age":26,"gender":"F","address":"290 Girard Street","employer":"Zomboid","email":"lovevelasquez@zomboid.com","city":"Villarreal","state":"SD","_is_real":true} +{"index":{"_id":"593"}} +{"account_number":593,"balance":41230,"firstname":"Muriel","lastname":"Vazquez","age":37,"gender":"M","address":"395 Montgomery Street","employer":"Sustenza","email":"murielvazquez@sustenza.com","city":"Strykersville","state":"OK","_is_real":true} +{"index":{"_id":"598"}} +{"account_number":598,"balance":33251,"firstname":"Morgan","lastname":"Coleman","age":33,"gender":"M","address":"324 McClancy Place","employer":"Aclima","email":"morgancoleman@aclima.com","city":"Bowden","state":"WA","_is_real":true} +{"index":{"_id":"601"}} +{"account_number":601,"balance":20796,"firstname":"Vickie","lastname":"Valentine","age":34,"gender":"F","address":"432 Bassett Avenue","employer":"Comvene","email":"vickievalentine@comvene.com","city":"Teasdale","state":"UT","_is_real":true} +{"index":{"_id":"606"}} +{"account_number":606,"balance":28770,"firstname":"Michael","lastname":"Bray","age":31,"gender":"M","address":"935 Lake Place","employer":"Telepark","email":"michaelbray@telepark.com","city":"Lemoyne","state":"CT","_is_real":true} +{"index":{"_id":"613"}} +{"account_number":613,"balance":39340,"firstname":"Eddie","lastname":"Mccarty","age":34,"gender":"F","address":"971 Richards Street","employer":"Bisba","email":"eddiemccarty@bisba.com","city":"Fruitdale","state":"NY","_is_real":true} +{"index":{"_id":"618"}} +{"account_number":618,"balance":8976,"firstname":"Cheri","lastname":"Ford","age":30,"gender":"F","address":"803 Ridgewood Avenue","employer":"Zorromop","email":"cheriford@zorromop.com","city":"Gambrills","state":"VT","_is_real":true} +{"index":{"_id":"620"}} +{"account_number":620,"balance":7224,"firstname":"Coleen","lastname":"Bartlett","age":38,"gender":"M","address":"761 Carroll Street","employer":"Idealis","email":"coleenbartlett@idealis.com","city":"Mathews","state":"DE","_is_real":true} +{"index":{"_id":"625"}} +{"account_number":625,"balance":46010,"firstname":"Cynthia","lastname":"Johnston","age":23,"gender":"M","address":"142 Box Street","employer":"Zentry","email":"cynthiajohnston@zentry.com","city":"Makena","state":"MA","_is_real":true} +{"index":{"_id":"632"}} +{"account_number":632,"balance":40470,"firstname":"Kay","lastname":"Warren","age":20,"gender":"F","address":"422 Alabama Avenue","employer":"Realysis","email":"kaywarren@realysis.com","city":"Homestead","state":"HI","_is_real":true} +{"index":{"_id":"637"}} +{"account_number":637,"balance":3169,"firstname":"Kathy","lastname":"Carter","age":27,"gender":"F","address":"410 Jamison Lane","employer":"Limage","email":"kathycarter@limage.com","city":"Ernstville","state":"WA","_is_real":true} +{"index":{"_id":"644"}} +{"account_number":644,"balance":44021,"firstname":"Etta","lastname":"Miller","age":21,"gender":"F","address":"376 Lawton Street","employer":"Bluegrain","email":"ettamiller@bluegrain.com","city":"Baker","state":"MD","_is_real":true} +{"index":{"_id":"649"}} +{"account_number":649,"balance":20275,"firstname":"Jeanine","lastname":"Malone","age":26,"gender":"F","address":"114 Dodworth Street","employer":"Nixelt","email":"jeaninemalone@nixelt.com","city":"Keyport","state":"AK","_is_real":true} +{"index":{"_id":"651"}} +{"account_number":651,"balance":18360,"firstname":"Young","lastname":"Reeves","age":34,"gender":"M","address":"581 Plaza Street","employer":"Krog","email":"youngreeves@krog.com","city":"Sussex","state":"WY","_is_real":true} +{"index":{"_id":"656"}} +{"account_number":656,"balance":21632,"firstname":"Olson","lastname":"Hunt","age":36,"gender":"M","address":"342 Jaffray Street","employer":"Volax","email":"olsonhunt@volax.com","city":"Bangor","state":"WA","_is_real":true} +{"index":{"_id":"663"}} +{"account_number":663,"balance":2456,"firstname":"Rollins","lastname":"Richards","age":37,"gender":"M","address":"129 Sullivan Place","employer":"Geostele","email":"rollinsrichards@geostele.com","city":"Morgandale","state":"FL","_is_real":true} +{"index":{"_id":"668"}} +{"account_number":668,"balance":45069,"firstname":"Potter","lastname":"Michael","age":27,"gender":"M","address":"803 Glenmore Avenue","employer":"Ontality","email":"pottermichael@ontality.com","city":"Newkirk","state":"KS","_is_real":true} +{"index":{"_id":"670"}} +{"account_number":670,"balance":10178,"firstname":"Ollie","lastname":"Riley","age":22,"gender":"M","address":"252 Jackson Place","employer":"Adornica","email":"ollieriley@adornica.com","city":"Brethren","state":"WI","_is_real":true} +{"index":{"_id":"675"}} +{"account_number":675,"balance":36102,"firstname":"Fisher","lastname":"Shepard","age":27,"gender":"F","address":"859 Varick Street","employer":"Qot","email":"fishershepard@qot.com","city":"Diaperville","state":"MD","_is_real":true} +{"index":{"_id":"682"}} +{"account_number":682,"balance":14168,"firstname":"Anne","lastname":"Hale","age":22,"gender":"F","address":"708 Anthony Street","employer":"Cytrek","email":"annehale@cytrek.com","city":"Beechmont","state":"WV","_is_real":true} +{"index":{"_id":"687"}} +{"account_number":687,"balance":48630,"firstname":"Caroline","lastname":"Cox","age":31,"gender":"M","address":"626 Hillel Place","employer":"Opticon","email":"carolinecox@opticon.com","city":"Loma","state":"ND","_is_real":true} +{"index":{"_id":"694"}} +{"account_number":694,"balance":33125,"firstname":"Craig","lastname":"Palmer","age":31,"gender":"F","address":"273 Montrose Avenue","employer":"Comvey","email":"craigpalmer@comvey.com","city":"Cleary","state":"OK","_is_real":true} +{"index":{"_id":"699"}} +{"account_number":699,"balance":4156,"firstname":"Gallagher","lastname":"Marshall","age":37,"gender":"F","address":"648 Clifford Place","employer":"Exiand","email":"gallaghermarshall@exiand.com","city":"Belfair","state":"KY","_is_real":true} +{"index":{"_id":"702"}} +{"account_number":702,"balance":46490,"firstname":"Meadows","lastname":"Delgado","age":26,"gender":"M","address":"612 Jardine Place","employer":"Daisu","email":"meadowsdelgado@daisu.com","city":"Venice","state":"AR","_is_real":true} +{"index":{"_id":"707"}} +{"account_number":707,"balance":30325,"firstname":"Sonya","lastname":"Trevino","age":30,"gender":"F","address":"181 Irving Place","employer":"Atgen","email":"sonyatrevino@atgen.com","city":"Enetai","state":"TN","_is_real":true} +{"index":{"_id":"714"}} +{"account_number":714,"balance":16602,"firstname":"Socorro","lastname":"Murray","age":34,"gender":"F","address":"810 Manhattan Court","employer":"Isoswitch","email":"socorromurray@isoswitch.com","city":"Jugtown","state":"AZ","_is_real":true} +{"index":{"_id":"719"}} +{"account_number":719,"balance":33107,"firstname":"Leanna","lastname":"Reed","age":25,"gender":"F","address":"528 Krier Place","employer":"Rodeology","email":"leannareed@rodeology.com","city":"Carrizo","state":"WI","_is_real":true} +{"index":{"_id":"721"}} +{"account_number":721,"balance":32958,"firstname":"Mara","lastname":"Dickson","age":26,"gender":"M","address":"810 Harrison Avenue","employer":"Comtours","email":"maradickson@comtours.com","city":"Thynedale","state":"DE","_is_real":true} +{"index":{"_id":"726"}} +{"account_number":726,"balance":44737,"firstname":"Rosemary","lastname":"Salazar","age":21,"gender":"M","address":"290 Croton Loop","employer":"Rockabye","email":"rosemarysalazar@rockabye.com","city":"Helen","state":"IA","_is_real":true} +{"index":{"_id":"733"}} +{"account_number":733,"balance":15722,"firstname":"Lakeisha","lastname":"Mccarthy","age":37,"gender":"M","address":"782 Turnbull Avenue","employer":"Exosis","email":"lakeishamccarthy@exosis.com","city":"Caberfae","state":"NM","_is_real":true} +{"index":{"_id":"738"}} +{"account_number":738,"balance":44936,"firstname":"Rosalind","lastname":"Hunter","age":32,"gender":"M","address":"644 Eaton Court","employer":"Zolarity","email":"rosalindhunter@zolarity.com","city":"Cataract","state":"SD","_is_real":true} +{"index":{"_id":"740"}} +{"account_number":740,"balance":6143,"firstname":"Chambers","lastname":"Hahn","age":22,"gender":"M","address":"937 Windsor Place","employer":"Medalert","email":"chambershahn@medalert.com","city":"Dorneyville","state":"DC","_is_real":true} +{"index":{"_id":"745"}} +{"account_number":745,"balance":4572,"firstname":"Jacobs","lastname":"Sweeney","age":32,"gender":"M","address":"189 Lott Place","employer":"Comtent","email":"jacobssweeney@comtent.com","city":"Advance","state":"NJ","_is_real":true} +{"index":{"_id":"752"}} +{"account_number":752,"balance":14039,"firstname":"Jerry","lastname":"Rush","age":31,"gender":"M","address":"632 Dank Court","employer":"Ebidco","email":"jerryrush@ebidco.com","city":"Geyserville","state":"AR","_is_real":true} +{"index":{"_id":"757"}} +{"account_number":757,"balance":34628,"firstname":"Mccullough","lastname":"Moore","age":30,"gender":"F","address":"304 Hastings Street","employer":"Nikuda","email":"mcculloughmoore@nikuda.com","city":"Charco","state":"DC","_is_real":true} +{"index":{"_id":"764"}} +{"account_number":764,"balance":3728,"firstname":"Noemi","lastname":"Gill","age":30,"gender":"M","address":"427 Chester Street","employer":"Avit","email":"noemigill@avit.com","city":"Chesterfield","state":"AL","_is_real":true} +{"index":{"_id":"769"}} +{"account_number":769,"balance":15362,"firstname":"Francis","lastname":"Beck","age":28,"gender":"M","address":"454 Livingston Street","employer":"Furnafix","email":"francisbeck@furnafix.com","city":"Dunnavant","state":"HI","_is_real":true} +{"index":{"_id":"771"}} +{"account_number":771,"balance":32784,"firstname":"Jocelyn","lastname":"Boone","age":23,"gender":"M","address":"513 Division Avenue","employer":"Collaire","email":"jocelynboone@collaire.com","city":"Lisco","state":"VT","_is_real":true} +{"index":{"_id":"776"}} +{"account_number":776,"balance":29177,"firstname":"Duke","lastname":"Atkinson","age":24,"gender":"M","address":"520 Doscher Street","employer":"Tripsch","email":"dukeatkinson@tripsch.com","city":"Lafferty","state":"NC","_is_real":true} +{"index":{"_id":"783"}} +{"account_number":783,"balance":11911,"firstname":"Faith","lastname":"Cooper","age":25,"gender":"F","address":"539 Rapelye Street","employer":"Insuron","email":"faithcooper@insuron.com","city":"Jennings","state":"MN","_is_real":true} +{"index":{"_id":"788"}} +{"account_number":788,"balance":12473,"firstname":"Marianne","lastname":"Aguilar","age":39,"gender":"F","address":"213 Holly Street","employer":"Marqet","email":"marianneaguilar@marqet.com","city":"Alfarata","state":"HI","_is_real":true} +{"index":{"_id":"790"}} +{"account_number":790,"balance":29912,"firstname":"Ellis","lastname":"Sullivan","age":39,"gender":"F","address":"877 Coyle Street","employer":"Enersave","email":"ellissullivan@enersave.com","city":"Canby","state":"MS","_is_real":true} +{"index":{"_id":"795"}} +{"account_number":795,"balance":31450,"firstname":"Bruce","lastname":"Avila","age":34,"gender":"M","address":"865 Newkirk Placez","employer":"Plasmosis","email":"bruceavila@plasmosis.com","city":"Ada","state":"ID","_is_real":true} +{"index":{"_id":"803"}} +{"account_number":803,"balance":49567,"firstname":"Marissa","lastname":"Spears","age":25,"gender":"M","address":"963 Highland Avenue","employer":"Centregy","email":"marissaspears@centregy.com","city":"Bloomington","state":"MS","_is_real":true} +{"index":{"_id":"808"}} +{"account_number":808,"balance":11251,"firstname":"Nola","lastname":"Quinn","age":20,"gender":"M","address":"863 Wythe Place","employer":"Iplax","email":"nolaquinn@iplax.com","city":"Cuylerville","state":"NH","_is_real":true} +{"index":{"_id":"810"}} +{"account_number":810,"balance":10563,"firstname":"Alyssa","lastname":"Ortega","age":40,"gender":"M","address":"977 Clymer Street","employer":"Eventage","email":"alyssaortega@eventage.com","city":"Convent","state":"SC","_is_real":true} +{"index":{"_id":"815"}} +{"account_number":815,"balance":19336,"firstname":"Guthrie","lastname":"Morse","age":30,"gender":"M","address":"685 Vandalia Avenue","employer":"Gronk","email":"guthriemorse@gronk.com","city":"Fowlerville","state":"OR","_is_real":true} +{"index":{"_id":"822"}} +{"account_number":822,"balance":13024,"firstname":"Hicks","lastname":"Farrell","age":25,"gender":"M","address":"468 Middleton Street","employer":"Zolarex","email":"hicksfarrell@zolarex.com","city":"Columbus","state":"OR","_is_real":true} +{"index":{"_id":"827"}} +{"account_number":827,"balance":37536,"firstname":"Naomi","lastname":"Ball","age":29,"gender":"F","address":"319 Stewart Street","employer":"Isotronic","email":"naomiball@isotronic.com","city":"Trona","state":"NM","_is_real":true} +{"index":{"_id":"834"}} +{"account_number":834,"balance":38049,"firstname":"Sybil","lastname":"Carrillo","age":25,"gender":"M","address":"359 Baughman Place","employer":"Phuel","email":"sybilcarrillo@phuel.com","city":"Kohatk","state":"CT","_is_real":true} +{"index":{"_id":"839"}} +{"account_number":839,"balance":38292,"firstname":"Langley","lastname":"Neal","age":39,"gender":"F","address":"565 Newton Street","employer":"Liquidoc","email":"langleyneal@liquidoc.com","city":"Osage","state":"AL","_is_real":true} +{"index":{"_id":"841"}} +{"account_number":841,"balance":28291,"firstname":"Dalton","lastname":"Waters","age":21,"gender":"M","address":"859 Grand Street","employer":"Malathion","email":"daltonwaters@malathion.com","city":"Tonopah","state":"AZ","_is_real":true} +{"index":{"_id":"846"}} +{"account_number":846,"balance":35099,"firstname":"Maureen","lastname":"Glass","age":22,"gender":"M","address":"140 Amherst Street","employer":"Stelaecor","email":"maureenglass@stelaecor.com","city":"Cucumber","state":"IL","_is_real":true} +{"index":{"_id":"853"}} +{"account_number":853,"balance":38353,"firstname":"Travis","lastname":"Parks","age":40,"gender":"M","address":"930 Bay Avenue","employer":"Pyramax","email":"travisparks@pyramax.com","city":"Gadsden","state":"ND","_is_real":true} +{"index":{"_id":"858"}} +{"account_number":858,"balance":23194,"firstname":"Small","lastname":"Hatfield","age":36,"gender":"M","address":"593 Tennis Court","employer":"Letpro","email":"smallhatfield@letpro.com","city":"Haena","state":"KS","_is_real":true} +{"index":{"_id":"860"}} +{"account_number":860,"balance":23613,"firstname":"Clark","lastname":"Boyd","age":37,"gender":"M","address":"501 Rock Street","employer":"Deepends","email":"clarkboyd@deepends.com","city":"Whitewater","state":"MA","_is_real":true} +{"index":{"_id":"865"}} +{"account_number":865,"balance":10574,"firstname":"Cook","lastname":"Kelley","age":28,"gender":"F","address":"865 Lincoln Terrace","employer":"Quizmo","email":"cookkelley@quizmo.com","city":"Kansas","state":"KY","_is_real":true} +{"index":{"_id":"872"}} +{"account_number":872,"balance":26314,"firstname":"Jane","lastname":"Greer","age":36,"gender":"F","address":"717 Hewes Street","employer":"Newcube","email":"janegreer@newcube.com","city":"Delshire","state":"DE","_is_real":true} +{"index":{"_id":"877"}} +{"account_number":877,"balance":42879,"firstname":"Tracey","lastname":"Ruiz","age":34,"gender":"F","address":"141 Tompkins Avenue","employer":"Waab","email":"traceyruiz@waab.com","city":"Zeba","state":"NM","_is_real":true} +{"index":{"_id":"884"}} +{"account_number":884,"balance":29316,"firstname":"Reva","lastname":"Rosa","age":40,"gender":"M","address":"784 Greene Avenue","employer":"Urbanshee","email":"revarosa@urbanshee.com","city":"Bakersville","state":"MS","_is_real":true} +{"index":{"_id":"889"}} +{"account_number":889,"balance":26464,"firstname":"Fischer","lastname":"Klein","age":38,"gender":"F","address":"948 Juliana Place","employer":"Comtext","email":"fischerklein@comtext.com","city":"Jackpot","state":"PA","_is_real":true} +{"index":{"_id":"891"}} +{"account_number":891,"balance":34829,"firstname":"Jacobson","lastname":"Clemons","age":24,"gender":"F","address":"507 Wilson Street","employer":"Quilm","email":"jacobsonclemons@quilm.com","city":"Muir","state":"TX","_is_real":true} +{"index":{"_id":"896"}} +{"account_number":896,"balance":31947,"firstname":"Buckley","lastname":"Peterson","age":26,"gender":"M","address":"217 Beayer Place","employer":"Earwax","email":"buckleypeterson@earwax.com","city":"Franklin","state":"DE","_is_real":true} +{"index":{"_id":"904"}} +{"account_number":904,"balance":27707,"firstname":"Mendez","lastname":"Mcneil","age":26,"gender":"M","address":"431 Halsey Street","employer":"Macronaut","email":"mendezmcneil@macronaut.com","city":"Troy","state":"OK","_is_real":true} +{"index":{"_id":"909"}} +{"account_number":909,"balance":18421,"firstname":"Stark","lastname":"Lewis","age":36,"gender":"M","address":"409 Tilden Avenue","employer":"Frosnex","email":"starklewis@frosnex.com","city":"Axis","state":"CA","_is_real":true} +{"index":{"_id":"911"}} +{"account_number":911,"balance":42655,"firstname":"Annie","lastname":"Lyons","age":21,"gender":"M","address":"518 Woods Place","employer":"Enerforce","email":"annielyons@enerforce.com","city":"Stagecoach","state":"MA","_is_real":true} +{"index":{"_id":"916"}} +{"account_number":916,"balance":47887,"firstname":"Jarvis","lastname":"Alexander","age":40,"gender":"M","address":"406 Bergen Avenue","employer":"Equitax","email":"jarvisalexander@equitax.com","city":"Haring","state":"KY","_is_real":true} +{"index":{"_id":"923"}} +{"account_number":923,"balance":48466,"firstname":"Mueller","lastname":"Mckee","age":26,"gender":"M","address":"298 Ruby Street","employer":"Luxuria","email":"muellermckee@luxuria.com","city":"Coleville","state":"TN","_is_real":true} +{"index":{"_id":"928"}} +{"account_number":928,"balance":19611,"firstname":"Hester","lastname":"Copeland","age":22,"gender":"F","address":"425 Cropsey Avenue","employer":"Dymi","email":"hestercopeland@dymi.com","city":"Wolcott","state":"NE","_is_real":true} +{"index":{"_id":"930"}} +{"account_number":930,"balance":47257,"firstname":"Kinney","lastname":"Lawson","age":39,"gender":"M","address":"501 Raleigh Place","employer":"Neptide","email":"kinneylawson@neptide.com","city":"Deltaville","state":"MD","_is_real":true} +{"index":{"_id":"935"}} +{"account_number":935,"balance":4959,"firstname":"Flowers","lastname":"Robles","age":30,"gender":"M","address":"201 Hull Street","employer":"Xelegyl","email":"flowersrobles@xelegyl.com","city":"Rehrersburg","state":"AL","_is_real":true} +{"index":{"_id":"942"}} +{"account_number":942,"balance":21299,"firstname":"Hamilton","lastname":"Clayton","age":26,"gender":"M","address":"413 Debevoise Street","employer":"Architax","email":"hamiltonclayton@architax.com","city":"Terlingua","state":"NM","_is_real":true} +{"index":{"_id":"947"}} +{"account_number":947,"balance":22039,"firstname":"Virgie","lastname":"Garza","age":30,"gender":"M","address":"903 Matthews Court","employer":"Plasmox","email":"virgiegarza@plasmox.com","city":"Somerset","state":"WY","_is_real":true} +{"index":{"_id":"954"}} +{"account_number":954,"balance":49404,"firstname":"Jenna","lastname":"Martin","age":22,"gender":"M","address":"688 Hart Street","employer":"Zinca","email":"jennamartin@zinca.com","city":"Oasis","state":"MD","_is_real":true} +{"index":{"_id":"959"}} +{"account_number":959,"balance":34743,"firstname":"Shaffer","lastname":"Cervantes","age":40,"gender":"M","address":"931 Varick Avenue","employer":"Oceanica","email":"shaffercervantes@oceanica.com","city":"Bowie","state":"AL","_is_real":true} +{"index":{"_id":"961"}} +{"account_number":961,"balance":43219,"firstname":"Betsy","lastname":"Hyde","age":27,"gender":"F","address":"183 Junius Street","employer":"Tubalum","email":"betsyhyde@tubalum.com","city":"Driftwood","state":"TX","_is_real":true} +{"index":{"_id":"966"}} +{"account_number":966,"balance":20619,"firstname":"Susanne","lastname":"Rodriguez","age":35,"gender":"F","address":"255 Knickerbocker Avenue","employer":"Comtrek","email":"susannerodriguez@comtrek.com","city":"Trinway","state":"TX","_is_real":true} +{"index":{"_id":"973"}} +{"account_number":973,"balance":45756,"firstname":"Rice","lastname":"Farmer","age":31,"gender":"M","address":"476 Nassau Avenue","employer":"Photobin","email":"ricefarmer@photobin.com","city":"Suitland","state":"ME","_is_real":true} +{"index":{"_id":"978"}} +{"account_number":978,"balance":21459,"firstname":"Melanie","lastname":"Rojas","age":33,"gender":"M","address":"991 Java Street","employer":"Kage","email":"melanierojas@kage.com","city":"Greenock","state":"VT","_is_real":true} +{"index":{"_id":"980"}} +{"account_number":980,"balance":42436,"firstname":"Cash","lastname":"Collier","age":33,"gender":"F","address":"999 Sapphire Street","employer":"Ceprene","email":"cashcollier@ceprene.com","city":"Glidden","state":"AK","_is_real":true} +{"index":{"_id":"985"}} +{"account_number":985,"balance":20083,"firstname":"Martin","lastname":"Gardner","age":28,"gender":"F","address":"644 Fairview Place","employer":"Golistic","email":"martingardner@golistic.com","city":"Connerton","state":"NJ","_is_real":true} +{"index":{"_id":"992"}} +{"account_number":992,"balance":11413,"firstname":"Kristie","lastname":"Kennedy","age":33,"gender":"F","address":"750 Hudson Avenue","employer":"Ludak","email":"kristiekennedy@ludak.com","city":"Warsaw","state":"WY","_is_real":true} +{"index":{"_id":"997"}} +{"account_number":997,"balance":25311,"firstname":"Combs","lastname":"Frederick","age":20,"gender":"M","address":"586 Lloyd Court","employer":"Pathways","email":"combsfrederick@pathways.com","city":"Williamson","state":"CA","_is_real":true} +{"index":{"_id":"3"}} +{"account_number":3,"balance":44947,"firstname":"Levine","lastname":"Burks","age":26,"gender":"F","address":"328 Wilson Avenue","employer":"Amtap","email":"levineburks@amtap.com","city":"Cochranville","state":"HI","_is_real":true} +{"index":{"_id":"8"}} +{"account_number":8,"balance":48868,"firstname":"Jan","lastname":"Burns","age":35,"gender":"M","address":"699 Visitation Place","employer":"Glasstep","email":"janburns@glasstep.com","city":"Wakulla","state":"AZ","_is_real":true} +{"index":{"_id":"10"}} +{"account_number":10,"balance":46170,"firstname":"Dominique","lastname":"Park","age":37,"gender":"F","address":"100 Gatling Place","employer":"Conjurica","email":"dominiquepark@conjurica.com","city":"Omar","state":"NJ","_is_real":true} +{"index":{"_id":"15"}} +{"account_number":15,"balance":43456,"firstname":"Bobbie","lastname":"Sexton","age":21,"gender":"M","address":"232 Sedgwick Place","employer":"Zytrex","email":"bobbiesexton@zytrex.com","city":"Hendersonville","state":"CA","_is_real":true} +{"index":{"_id":"22"}} +{"account_number":22,"balance":40283,"firstname":"Barrera","lastname":"Terrell","age":23,"gender":"F","address":"292 Orange Street","employer":"Steelfab","email":"barreraterrell@steelfab.com","city":"Bynum","state":"ME","_is_real":true} +{"index":{"_id":"27"}} +{"account_number":27,"balance":6176,"firstname":"Meyers","lastname":"Williamson","age":26,"gender":"F","address":"675 Henderson Walk","employer":"Plexia","email":"meyerswilliamson@plexia.com","city":"Richmond","state":"AZ","_is_real":true} +{"index":{"_id":"34"}} +{"account_number":34,"balance":35379,"firstname":"Ellison","lastname":"Kim","age":30,"gender":"F","address":"986 Revere Place","employer":"Signity","email":"ellisonkim@signity.com","city":"Sehili","state":"IL","_is_real":true} +{"index":{"_id":"39"}} +{"account_number":39,"balance":38688,"firstname":"Bowers","lastname":"Mendez","age":22,"gender":"F","address":"665 Bennet Court","employer":"Farmage","email":"bowersmendez@farmage.com","city":"Duryea","state":"PA","_is_real":true} +{"index":{"_id":"41"}} +{"account_number":41,"balance":36060,"firstname":"Hancock","lastname":"Holden","age":20,"gender":"M","address":"625 Gaylord Drive","employer":"Poochies","email":"hancockholden@poochies.com","city":"Alamo","state":"KS","_is_real":true} +{"index":{"_id":"46"}} +{"account_number":46,"balance":12351,"firstname":"Karla","lastname":"Bowman","age":23,"gender":"M","address":"554 Chapel Street","employer":"Undertap","email":"karlabowman@undertap.com","city":"Sylvanite","state":"DC","_is_real":true} +{"index":{"_id":"53"}} +{"account_number":53,"balance":28101,"firstname":"Kathryn","lastname":"Payne","age":29,"gender":"F","address":"467 Louis Place","employer":"Katakana","email":"kathrynpayne@katakana.com","city":"Harviell","state":"SD","_is_real":true} +{"index":{"_id":"58"}} +{"account_number":58,"balance":31697,"firstname":"Marva","lastname":"Cannon","age":40,"gender":"M","address":"993 Highland Place","employer":"Comcubine","email":"marvacannon@comcubine.com","city":"Orviston","state":"MO","_is_real":true} +{"index":{"_id":"60"}} +{"account_number":60,"balance":45955,"firstname":"Maude","lastname":"Casey","age":31,"gender":"F","address":"566 Strauss Street","employer":"Quilch","email":"maudecasey@quilch.com","city":"Enlow","state":"GA","_is_real":true} +{"index":{"_id":"65"}} +{"account_number":65,"balance":23282,"firstname":"Leonor","lastname":"Pruitt","age":24,"gender":"M","address":"974 Terrace Place","employer":"Velos","email":"leonorpruitt@velos.com","city":"Devon","state":"WI","_is_real":true} +{"index":{"_id":"72"}} +{"account_number":72,"balance":9732,"firstname":"Barlow","lastname":"Rhodes","age":25,"gender":"F","address":"891 Clinton Avenue","employer":"Zialactic","email":"barlowrhodes@zialactic.com","city":"Echo","state":"TN","_is_real":true} +{"index":{"_id":"77"}} +{"account_number":77,"balance":5724,"firstname":"Byrd","lastname":"Conley","age":24,"gender":"F","address":"698 Belmont Avenue","employer":"Zidox","email":"byrdconley@zidox.com","city":"Rockbridge","state":"SC","_is_real":true} +{"index":{"_id":"84"}} +{"account_number":84,"balance":3001,"firstname":"Hutchinson","lastname":"Newton","age":34,"gender":"F","address":"553 Locust Street","employer":"Zaggles","email":"hutchinsonnewton@zaggles.com","city":"Snyderville","state":"DC","_is_real":true} +{"index":{"_id":"89"}} +{"account_number":89,"balance":13263,"firstname":"Mcdowell","lastname":"Bradley","age":28,"gender":"M","address":"960 Howard Alley","employer":"Grok","email":"mcdowellbradley@grok.com","city":"Toftrees","state":"TX","_is_real":true} +{"index":{"_id":"91"}} +{"account_number":91,"balance":29799,"firstname":"Vonda","lastname":"Galloway","age":20,"gender":"M","address":"988 Voorhies Avenue","employer":"Illumity","email":"vondagalloway@illumity.com","city":"Holcombe","state":"HI","_is_real":true} +{"index":{"_id":"96"}} +{"account_number":96,"balance":15933,"firstname":"Shirley","lastname":"Edwards","age":38,"gender":"M","address":"817 Caton Avenue","employer":"Equitox","email":"shirleyedwards@equitox.com","city":"Nelson","state":"MA","_is_real":true} +{"index":{"_id":"104"}} +{"account_number":104,"balance":32619,"firstname":"Casey","lastname":"Roth","age":29,"gender":"M","address":"963 Railroad Avenue","employer":"Hotcakes","email":"caseyroth@hotcakes.com","city":"Davenport","state":"OH","_is_real":true} +{"index":{"_id":"109"}} +{"account_number":109,"balance":25812,"firstname":"Gretchen","lastname":"Dawson","age":31,"gender":"M","address":"610 Bethel Loop","employer":"Tetak","email":"gretchendawson@tetak.com","city":"Hailesboro","state":"CO","_is_real":true} +{"index":{"_id":"111"}} +{"account_number":111,"balance":1481,"firstname":"Traci","lastname":"Allison","age":35,"gender":"M","address":"922 Bryant Street","employer":"Enjola","email":"traciallison@enjola.com","city":"Robinette","state":"OR","_is_real":true} +{"index":{"_id":"116"}} +{"account_number":116,"balance":21335,"firstname":"Hobbs","lastname":"Wright","age":24,"gender":"M","address":"965 Temple Court","employer":"Netbook","email":"hobbswright@netbook.com","city":"Strong","state":"CA","_is_real":true} +{"index":{"_id":"123"}} +{"account_number":123,"balance":3079,"firstname":"Cleo","lastname":"Beach","age":27,"gender":"F","address":"653 Haring Street","employer":"Proxsoft","email":"cleobeach@proxsoft.com","city":"Greensburg","state":"ME","_is_real":true} +{"index":{"_id":"128"}} +{"account_number":128,"balance":3556,"firstname":"Mack","lastname":"Bullock","age":34,"gender":"F","address":"462 Ingraham Street","employer":"Terascape","email":"mackbullock@terascape.com","city":"Eureka","state":"PA","_is_real":true} +{"index":{"_id":"130"}} +{"account_number":130,"balance":24171,"firstname":"Roxie","lastname":"Cantu","age":33,"gender":"M","address":"841 Catherine Street","employer":"Skybold","email":"roxiecantu@skybold.com","city":"Deputy","state":"NE","_is_real":true} +{"index":{"_id":"135"}} +{"account_number":135,"balance":24885,"firstname":"Stevenson","lastname":"Crosby","age":40,"gender":"F","address":"473 Boardwalk ","employer":"Accel","email":"stevensoncrosby@accel.com","city":"Norris","state":"OK","_is_real":true} +{"index":{"_id":"142"}} +{"account_number":142,"balance":4544,"firstname":"Vang","lastname":"Hughes","age":27,"gender":"M","address":"357 Landis Court","employer":"Bolax","email":"vanghughes@bolax.com","city":"Emerald","state":"WY","_is_real":true} +{"index":{"_id":"147"}} +{"account_number":147,"balance":35921,"firstname":"Charmaine","lastname":"Whitney","age":28,"gender":"F","address":"484 Seton Place","employer":"Comveyer","email":"charmainewhitney@comveyer.com","city":"Dexter","state":"DC","_is_real":true} +{"index":{"_id":"154"}} +{"account_number":154,"balance":40945,"firstname":"Burns","lastname":"Solis","age":31,"gender":"M","address":"274 Lorraine Street","employer":"Rodemco","email":"burnssolis@rodemco.com","city":"Ballico","state":"WI","_is_real":true} +{"index":{"_id":"159"}} +{"account_number":159,"balance":1696,"firstname":"Alvarez","lastname":"Mack","age":22,"gender":"F","address":"897 Manor Court","employer":"Snorus","email":"alvarezmack@snorus.com","city":"Rosedale","state":"CA","_is_real":true} +{"index":{"_id":"161"}} +{"account_number":161,"balance":4659,"firstname":"Doreen","lastname":"Randall","age":37,"gender":"F","address":"178 Court Street","employer":"Calcula","email":"doreenrandall@calcula.com","city":"Belmont","state":"TX","_is_real":true} +{"index":{"_id":"166"}} +{"account_number":166,"balance":33847,"firstname":"Rutledge","lastname":"Rivas","age":23,"gender":"M","address":"352 Verona Street","employer":"Virxo","email":"rutledgerivas@virxo.com","city":"Brandermill","state":"NE","_is_real":true} +{"index":{"_id":"173"}} +{"account_number":173,"balance":5989,"firstname":"Whitley","lastname":"Blevins","age":32,"gender":"M","address":"127 Brooklyn Avenue","employer":"Pawnagra","email":"whitleyblevins@pawnagra.com","city":"Rodanthe","state":"ND","_is_real":true} +{"index":{"_id":"178"}} +{"account_number":178,"balance":36735,"firstname":"Clements","lastname":"Finley","age":39,"gender":"F","address":"270 Story Court","employer":"Imaginart","email":"clementsfinley@imaginart.com","city":"Lookingglass","state":"MN","_is_real":true} +{"index":{"_id":"180"}} +{"account_number":180,"balance":34236,"firstname":"Ursula","lastname":"Goodman","age":32,"gender":"F","address":"414 Clinton Street","employer":"Earthmark","email":"ursulagoodman@earthmark.com","city":"Rote","state":"AR","_is_real":true} +{"index":{"_id":"185"}} +{"account_number":185,"balance":43532,"firstname":"Laurel","lastname":"Cline","age":40,"gender":"M","address":"788 Fenimore Street","employer":"Prismatic","email":"laurelcline@prismatic.com","city":"Frank","state":"UT","_is_real":true} +{"index":{"_id":"192"}} +{"account_number":192,"balance":23508,"firstname":"Ramsey","lastname":"Carr","age":31,"gender":"F","address":"209 Williamsburg Street","employer":"Strezzo","email":"ramseycarr@strezzo.com","city":"Grapeview","state":"NM","_is_real":true} +{"index":{"_id":"197"}} +{"account_number":197,"balance":17246,"firstname":"Sweet","lastname":"Sanders","age":33,"gender":"F","address":"712 Homecrest Court","employer":"Isosure","email":"sweetsanders@isosure.com","city":"Sheatown","state":"VT","_is_real":true} +{"index":{"_id":"200"}} +{"account_number":200,"balance":26210,"firstname":"Teri","lastname":"Hester","age":39,"gender":"M","address":"653 Abbey Court","employer":"Electonic","email":"terihester@electonic.com","city":"Martell","state":"MD","_is_real":true} +{"index":{"_id":"205"}} +{"account_number":205,"balance":45493,"firstname":"Johnson","lastname":"Chang","age":28,"gender":"F","address":"331 John Street","employer":"Gleamink","email":"johnsonchang@gleamink.com","city":"Sultana","state":"KS","_is_real":true} +{"index":{"_id":"212"}} +{"account_number":212,"balance":10299,"firstname":"Marisol","lastname":"Fischer","age":39,"gender":"M","address":"362 Prince Street","employer":"Autograte","email":"marisolfischer@autograte.com","city":"Oley","state":"SC","_is_real":true} +{"index":{"_id":"217"}} +{"account_number":217,"balance":33730,"firstname":"Sally","lastname":"Mccoy","age":38,"gender":"F","address":"854 Corbin Place","employer":"Omnigog","email":"sallymccoy@omnigog.com","city":"Escondida","state":"FL","_is_real":true} +{"index":{"_id":"224"}} +{"account_number":224,"balance":42708,"firstname":"Billie","lastname":"Nixon","age":28,"gender":"F","address":"241 Kaufman Place","employer":"Xanide","email":"billienixon@xanide.com","city":"Chapin","state":"NY","_is_real":true} +{"index":{"_id":"229"}} +{"account_number":229,"balance":2740,"firstname":"Jana","lastname":"Hensley","age":30,"gender":"M","address":"176 Erasmus Street","employer":"Isotrack","email":"janahensley@isotrack.com","city":"Caledonia","state":"ME","_is_real":true} +{"index":{"_id":"231"}} +{"account_number":231,"balance":46180,"firstname":"Essie","lastname":"Clarke","age":34,"gender":"F","address":"308 Harbor Lane","employer":"Pharmacon","email":"essieclarke@pharmacon.com","city":"Fillmore","state":"MS","_is_real":true} +{"index":{"_id":"236"}} +{"account_number":236,"balance":41200,"firstname":"Suzanne","lastname":"Bird","age":39,"gender":"F","address":"219 Luquer Street","employer":"Imant","email":"suzannebird@imant.com","city":"Bainbridge","state":"NY","_is_real":true} +{"index":{"_id":"243"}} +{"account_number":243,"balance":29902,"firstname":"Evangelina","lastname":"Perez","age":20,"gender":"M","address":"787 Joval Court","employer":"Keengen","email":"evangelinaperez@keengen.com","city":"Mulberry","state":"SD","_is_real":true} +{"index":{"_id":"248"}} +{"account_number":248,"balance":49989,"firstname":"West","lastname":"England","age":36,"gender":"M","address":"717 Hendrickson Place","employer":"Obliq","email":"westengland@obliq.com","city":"Maury","state":"WA","_is_real":true} +{"index":{"_id":"250"}} +{"account_number":250,"balance":27893,"firstname":"Earlene","lastname":"Ellis","age":39,"gender":"F","address":"512 Bay Street","employer":"Codact","email":"earleneellis@codact.com","city":"Sunwest","state":"GA","_is_real":true} +{"index":{"_id":"255"}} +{"account_number":255,"balance":49339,"firstname":"Iva","lastname":"Rivers","age":38,"gender":"M","address":"470 Rost Place","employer":"Mantrix","email":"ivarivers@mantrix.com","city":"Disautel","state":"MD","_is_real":true} +{"index":{"_id":"262"}} +{"account_number":262,"balance":30289,"firstname":"Tameka","lastname":"Levine","age":36,"gender":"F","address":"815 Atlantic Avenue","employer":"Acium","email":"tamekalevine@acium.com","city":"Winchester","state":"SD","_is_real":true} +{"index":{"_id":"267"}} +{"account_number":267,"balance":42753,"firstname":"Weeks","lastname":"Castillo","age":21,"gender":"F","address":"526 Holt Court","employer":"Talendula","email":"weekscastillo@talendula.com","city":"Washington","state":"NV","_is_real":true} +{"index":{"_id":"274"}} +{"account_number":274,"balance":12104,"firstname":"Frieda","lastname":"House","age":33,"gender":"F","address":"171 Banker Street","employer":"Quonk","email":"friedahouse@quonk.com","city":"Aberdeen","state":"NJ","_is_real":true} +{"index":{"_id":"279"}} +{"account_number":279,"balance":15904,"firstname":"Chapman","lastname":"Hart","age":32,"gender":"F","address":"902 Bliss Terrace","employer":"Kongene","email":"chapmanhart@kongene.com","city":"Bradenville","state":"NJ","_is_real":true} +{"index":{"_id":"281"}} +{"account_number":281,"balance":39830,"firstname":"Bean","lastname":"Aguirre","age":20,"gender":"F","address":"133 Pilling Street","employer":"Amril","email":"beanaguirre@amril.com","city":"Waterview","state":"TX","_is_real":true} +{"index":{"_id":"286"}} +{"account_number":286,"balance":39063,"firstname":"Rosetta","lastname":"Turner","age":35,"gender":"M","address":"169 Jefferson Avenue","employer":"Spacewax","email":"rosettaturner@spacewax.com","city":"Stewart","state":"MO","_is_real":true} +{"index":{"_id":"293"}} +{"account_number":293,"balance":29867,"firstname":"Cruz","lastname":"Carver","age":28,"gender":"F","address":"465 Boerum Place","employer":"Vitricomp","email":"cruzcarver@vitricomp.com","city":"Crayne","state":"CO","_is_real":true} +{"index":{"_id":"298"}} +{"account_number":298,"balance":34334,"firstname":"Bullock","lastname":"Marsh","age":20,"gender":"M","address":"589 Virginia Place","employer":"Renovize","email":"bullockmarsh@renovize.com","city":"Coinjock","state":"UT","_is_real":true} +{"index":{"_id":"301"}} +{"account_number":301,"balance":16782,"firstname":"Minerva","lastname":"Graham","age":35,"gender":"M","address":"532 Harrison Place","employer":"Sureplex","email":"minervagraham@sureplex.com","city":"Belleview","state":"GA","_is_real":true} +{"index":{"_id":"306"}} +{"account_number":306,"balance":2171,"firstname":"Hensley","lastname":"Hardin","age":40,"gender":"M","address":"196 Maujer Street","employer":"Neocent","email":"hensleyhardin@neocent.com","city":"Reinerton","state":"HI","_is_real":true} +{"index":{"_id":"313"}} +{"account_number":313,"balance":34108,"firstname":"Alston","lastname":"Henderson","age":36,"gender":"F","address":"132 Prescott Place","employer":"Prosure","email":"alstonhenderson@prosure.com","city":"Worton","state":"IA","_is_real":true} +{"index":{"_id":"318"}} +{"account_number":318,"balance":8512,"firstname":"Nichole","lastname":"Pearson","age":34,"gender":"F","address":"656 Lacon Court","employer":"Yurture","email":"nicholepearson@yurture.com","city":"Juarez","state":"MO","_is_real":true} +{"index":{"_id":"320"}} +{"account_number":320,"balance":34521,"firstname":"Patti","lastname":"Brennan","age":37,"gender":"F","address":"870 Degraw Street","employer":"Cognicode","email":"pattibrennan@cognicode.com","city":"Torboy","state":"FL","_is_real":true} +{"index":{"_id":"325"}} +{"account_number":325,"balance":1956,"firstname":"Magdalena","lastname":"Simmons","age":25,"gender":"F","address":"681 Townsend Street","employer":"Geekosis","email":"magdalenasimmons@geekosis.com","city":"Sterling","state":"CA","_is_real":true} +{"index":{"_id":"332"}} +{"account_number":332,"balance":37770,"firstname":"Shepherd","lastname":"Davenport","age":28,"gender":"F","address":"586 Montague Terrace","employer":"Ecraze","email":"shepherddavenport@ecraze.com","city":"Accoville","state":"NM","_is_real":true} +{"index":{"_id":"337"}} +{"account_number":337,"balance":43432,"firstname":"Monroe","lastname":"Stafford","age":37,"gender":"F","address":"183 Seigel Street","employer":"Centuria","email":"monroestafford@centuria.com","city":"Camino","state":"DE","_is_real":true} +{"index":{"_id":"344"}} +{"account_number":344,"balance":42654,"firstname":"Sasha","lastname":"Baxter","age":35,"gender":"F","address":"700 Bedford Place","employer":"Callflex","email":"sashabaxter@callflex.com","city":"Campo","state":"MI","_is_real":true} +{"index":{"_id":"349"}} +{"account_number":349,"balance":24180,"firstname":"Allison","lastname":"Fitzpatrick","age":22,"gender":"F","address":"913 Arlington Avenue","employer":"Veraq","email":"allisonfitzpatrick@veraq.com","city":"Marbury","state":"TX","_is_real":true} +{"index":{"_id":"351"}} +{"account_number":351,"balance":47089,"firstname":"Hendrix","lastname":"Stephens","age":29,"gender":"M","address":"181 Beaver Street","employer":"Recrisys","email":"hendrixstephens@recrisys.com","city":"Denio","state":"OR","_is_real":true} +{"index":{"_id":"356"}} +{"account_number":356,"balance":34540,"firstname":"Lourdes","lastname":"Valdez","age":20,"gender":"F","address":"700 Anchorage Place","employer":"Interloo","email":"lourdesvaldez@interloo.com","city":"Goldfield","state":"OK","_is_real":true} +{"index":{"_id":"363"}} +{"account_number":363,"balance":34007,"firstname":"Peggy","lastname":"Bright","age":21,"gender":"M","address":"613 Engert Avenue","employer":"Inventure","email":"peggybright@inventure.com","city":"Chautauqua","state":"ME","_is_real":true} +{"index":{"_id":"368"}} +{"account_number":368,"balance":23535,"firstname":"Hooper","lastname":"Tyson","age":39,"gender":"M","address":"892 Taaffe Place","employer":"Zaggle","email":"hoopertyson@zaggle.com","city":"Nutrioso","state":"ME","_is_real":true} +{"index":{"_id":"370"}} +{"account_number":370,"balance":28499,"firstname":"Oneill","lastname":"Carney","age":25,"gender":"F","address":"773 Adelphi Street","employer":"Bedder","email":"oneillcarney@bedder.com","city":"Yorklyn","state":"FL","_is_real":true} +{"index":{"_id":"375"}} +{"account_number":375,"balance":23860,"firstname":"Phoebe","lastname":"Patton","age":25,"gender":"M","address":"564 Hale Avenue","employer":"Xoggle","email":"phoebepatton@xoggle.com","city":"Brule","state":"NM","_is_real":true} +{"index":{"_id":"382"}} +{"account_number":382,"balance":42061,"firstname":"Finley","lastname":"Singleton","age":37,"gender":"F","address":"407 Clay Street","employer":"Quarex","email":"finleysingleton@quarex.com","city":"Bedias","state":"LA","_is_real":true} +{"index":{"_id":"387"}} +{"account_number":387,"balance":35916,"firstname":"April","lastname":"Hill","age":29,"gender":"M","address":"818 Bayard Street","employer":"Kengen","email":"aprilhill@kengen.com","city":"Chloride","state":"NC","_is_real":true} +{"index":{"_id":"394"}} +{"account_number":394,"balance":6121,"firstname":"Lorrie","lastname":"Nunez","age":38,"gender":"M","address":"221 Ralph Avenue","employer":"Bullzone","email":"lorrienunez@bullzone.com","city":"Longoria","state":"ID","_is_real":true} +{"index":{"_id":"399"}} +{"account_number":399,"balance":32587,"firstname":"Carmela","lastname":"Franks","age":23,"gender":"M","address":"617 Dewey Place","employer":"Zensure","email":"carmelafranks@zensure.com","city":"Sanders","state":"DC","_is_real":true} +{"index":{"_id":"402"}} +{"account_number":402,"balance":1282,"firstname":"Pacheco","lastname":"Rosales","age":32,"gender":"M","address":"538 Pershing Loop","employer":"Circum","email":"pachecorosales@circum.com","city":"Elbert","state":"ID","_is_real":true} +{"index":{"_id":"407"}} +{"account_number":407,"balance":36417,"firstname":"Gilda","lastname":"Jacobson","age":29,"gender":"F","address":"883 Loring Avenue","employer":"Comveyor","email":"gildajacobson@comveyor.com","city":"Topaz","state":"NH","_is_real":true} +{"index":{"_id":"414"}} +{"account_number":414,"balance":17506,"firstname":"Conway","lastname":"Daugherty","age":37,"gender":"F","address":"643 Kermit Place","employer":"Lyria","email":"conwaydaugherty@lyria.com","city":"Vaughn","state":"NV","_is_real":true} +{"index":{"_id":"419"}} +{"account_number":419,"balance":34847,"firstname":"Helen","lastname":"Montoya","age":29,"gender":"F","address":"736 Kingsland Avenue","employer":"Hairport","email":"helenmontoya@hairport.com","city":"Edinburg","state":"NE","_is_real":true} +{"index":{"_id":"421"}} +{"account_number":421,"balance":46868,"firstname":"Tamika","lastname":"Mccall","age":27,"gender":"F","address":"764 Bragg Court","employer":"Eventix","email":"tamikamccall@eventix.com","city":"Tivoli","state":"RI","_is_real":true} +{"index":{"_id":"426"}} +{"account_number":426,"balance":4499,"firstname":"Julie","lastname":"Parsons","age":31,"gender":"M","address":"768 Keap Street","employer":"Goko","email":"julieparsons@goko.com","city":"Coldiron","state":"VA","_is_real":true} +{"index":{"_id":"433"}} +{"account_number":433,"balance":19266,"firstname":"Wilkinson","lastname":"Flowers","age":39,"gender":"M","address":"154 Douglass Street","employer":"Xsports","email":"wilkinsonflowers@xsports.com","city":"Coultervillle","state":"MN","_is_real":true} +{"index":{"_id":"438"}} +{"account_number":438,"balance":16367,"firstname":"Walter","lastname":"Velez","age":27,"gender":"F","address":"931 Farragut Road","employer":"Virva","email":"waltervelez@virva.com","city":"Tyro","state":"WV","_is_real":true} +{"index":{"_id":"440"}} +{"account_number":440,"balance":41590,"firstname":"Ray","lastname":"Wiley","age":31,"gender":"F","address":"102 Barwell Terrace","employer":"Polaria","email":"raywiley@polaria.com","city":"Hardyville","state":"IA","_is_real":true} +{"index":{"_id":"445"}} +{"account_number":445,"balance":41178,"firstname":"Rodriguez","lastname":"Macias","age":34,"gender":"M","address":"164 Boerum Street","employer":"Xylar","email":"rodriguezmacias@xylar.com","city":"Riner","state":"AL","_is_real":true} +{"index":{"_id":"452"}} +{"account_number":452,"balance":3589,"firstname":"Blackwell","lastname":"Delaney","age":39,"gender":"F","address":"443 Sackett Street","employer":"Imkan","email":"blackwelldelaney@imkan.com","city":"Gasquet","state":"DC","_is_real":true} +{"index":{"_id":"457"}} +{"account_number":457,"balance":14057,"firstname":"Bush","lastname":"Gordon","age":34,"gender":"M","address":"975 Dakota Place","employer":"Softmicro","email":"bushgordon@softmicro.com","city":"Chemung","state":"PA","_is_real":true} +{"index":{"_id":"464"}} +{"account_number":464,"balance":20504,"firstname":"Cobb","lastname":"Humphrey","age":21,"gender":"M","address":"823 Sunnyside Avenue","employer":"Apexia","email":"cobbhumphrey@apexia.com","city":"Wintersburg","state":"NY","_is_real":true} +{"index":{"_id":"469"}} +{"account_number":469,"balance":26509,"firstname":"Marci","lastname":"Shepherd","age":26,"gender":"M","address":"565 Hall Street","employer":"Shadease","email":"marcishepherd@shadease.com","city":"Springhill","state":"IL","_is_real":true} +{"index":{"_id":"471"}} +{"account_number":471,"balance":7629,"firstname":"Juana","lastname":"Silva","age":36,"gender":"M","address":"249 Amity Street","employer":"Artworlds","email":"juanasilva@artworlds.com","city":"Norfolk","state":"TX","_is_real":true} +{"index":{"_id":"476"}} +{"account_number":476,"balance":33386,"firstname":"Silva","lastname":"Marks","age":31,"gender":"F","address":"183 Eldert Street","employer":"Medifax","email":"silvamarks@medifax.com","city":"Hachita","state":"RI","_is_real":true} +{"index":{"_id":"483"}} +{"account_number":483,"balance":6344,"firstname":"Kelley","lastname":"Harper","age":29,"gender":"M","address":"758 Preston Court","employer":"Xyqag","email":"kelleyharper@xyqag.com","city":"Healy","state":"IA","_is_real":true} +{"index":{"_id":"488"}} +{"account_number":488,"balance":6289,"firstname":"Wilma","lastname":"Hopkins","age":38,"gender":"M","address":"428 Lee Avenue","employer":"Entality","email":"wilmahopkins@entality.com","city":"Englevale","state":"WI","_is_real":true} +{"index":{"_id":"490"}} +{"account_number":490,"balance":1447,"firstname":"Strong","lastname":"Hendrix","age":26,"gender":"F","address":"134 Beach Place","employer":"Duoflex","email":"stronghendrix@duoflex.com","city":"Allentown","state":"ND","_is_real":true} +{"index":{"_id":"495"}} +{"account_number":495,"balance":13478,"firstname":"Abigail","lastname":"Nichols","age":40,"gender":"F","address":"887 President Street","employer":"Enquility","email":"abigailnichols@enquility.com","city":"Bagtown","state":"NM","_is_real":true} +{"index":{"_id":"503"}} +{"account_number":503,"balance":42649,"firstname":"Leta","lastname":"Stout","age":39,"gender":"F","address":"518 Bowery Street","employer":"Pivitol","email":"letastout@pivitol.com","city":"Boonville","state":"ND","_is_real":true} +{"index":{"_id":"508"}} +{"account_number":508,"balance":41300,"firstname":"Lawrence","lastname":"Mathews","age":27,"gender":"F","address":"987 Rose Street","employer":"Deviltoe","email":"lawrencemathews@deviltoe.com","city":"Woodburn","state":"FL","_is_real":true} +{"index":{"_id":"510"}} +{"account_number":510,"balance":48504,"firstname":"Petty","lastname":"Sykes","age":28,"gender":"M","address":"566 Village Road","employer":"Nebulean","email":"pettysykes@nebulean.com","city":"Wedgewood","state":"MO","_is_real":true} +{"index":{"_id":"515"}} +{"account_number":515,"balance":18531,"firstname":"Lott","lastname":"Keller","age":27,"gender":"M","address":"827 Miami Court","employer":"Translink","email":"lottkeller@translink.com","city":"Gila","state":"TX","_is_real":true} +{"index":{"_id":"522"}} +{"account_number":522,"balance":19879,"firstname":"Faulkner","lastname":"Garrett","age":29,"gender":"F","address":"396 Grove Place","employer":"Pigzart","email":"faulknergarrett@pigzart.com","city":"Felt","state":"AR","_is_real":true} +{"index":{"_id":"527"}} +{"account_number":527,"balance":2028,"firstname":"Carver","lastname":"Peters","age":35,"gender":"M","address":"816 Victor Road","employer":"Housedown","email":"carverpeters@housedown.com","city":"Nadine","state":"MD","_is_real":true} +{"index":{"_id":"534"}} +{"account_number":534,"balance":20470,"firstname":"Cristina","lastname":"Russo","age":25,"gender":"F","address":"500 Highlawn Avenue","employer":"Cyclonica","email":"cristinarusso@cyclonica.com","city":"Gorst","state":"KS","_is_real":true} +{"index":{"_id":"539"}} +{"account_number":539,"balance":24560,"firstname":"Tami","lastname":"Maddox","age":23,"gender":"F","address":"741 Pineapple Street","employer":"Accidency","email":"tamimaddox@accidency.com","city":"Kennedyville","state":"OH","_is_real":true} +{"index":{"_id":"541"}} +{"account_number":541,"balance":42915,"firstname":"Logan","lastname":"Burke","age":32,"gender":"M","address":"904 Clarendon Road","employer":"Overplex","email":"loganburke@overplex.com","city":"Johnsonburg","state":"OH","_is_real":true} +{"index":{"_id":"546"}} +{"account_number":546,"balance":43242,"firstname":"Bernice","lastname":"Sims","age":33,"gender":"M","address":"382 Columbia Street","employer":"Verbus","email":"bernicesims@verbus.com","city":"Sena","state":"KY","_is_real":true} +{"index":{"_id":"553"}} +{"account_number":553,"balance":28390,"firstname":"Aimee","lastname":"Cohen","age":28,"gender":"M","address":"396 Lafayette Avenue","employer":"Eplode","email":"aimeecohen@eplode.com","city":"Thatcher","state":"NJ","_is_real":true} +{"index":{"_id":"558"}} +{"account_number":558,"balance":8922,"firstname":"Horne","lastname":"Valenzuela","age":20,"gender":"F","address":"979 Kensington Street","employer":"Isoternia","email":"hornevalenzuela@isoternia.com","city":"Greenbush","state":"NC","_is_real":true} +{"index":{"_id":"560"}} +{"account_number":560,"balance":24514,"firstname":"Felecia","lastname":"Oneill","age":26,"gender":"M","address":"995 Autumn Avenue","employer":"Mediot","email":"feleciaoneill@mediot.com","city":"Joppa","state":"IN","_is_real":true} +{"index":{"_id":"565"}} +{"account_number":565,"balance":15197,"firstname":"Taylor","lastname":"Ingram","age":37,"gender":"F","address":"113 Will Place","employer":"Lyrichord","email":"tayloringram@lyrichord.com","city":"Collins","state":"ME","_is_real":true} +{"index":{"_id":"572"}} +{"account_number":572,"balance":49355,"firstname":"Therese","lastname":"Espinoza","age":20,"gender":"M","address":"994 Chester Court","employer":"Gonkle","email":"thereseespinoza@gonkle.com","city":"Hayes","state":"UT","_is_real":true} +{"index":{"_id":"577"}} +{"account_number":577,"balance":21398,"firstname":"Gilbert","lastname":"Serrano","age":38,"gender":"F","address":"294 Troutman Street","employer":"Senmao","email":"gilbertserrano@senmao.com","city":"Greer","state":"MT","_is_real":true} +{"index":{"_id":"584"}} +{"account_number":584,"balance":5346,"firstname":"Pearson","lastname":"Bryant","age":40,"gender":"F","address":"971 Heyward Street","employer":"Anacho","email":"pearsonbryant@anacho.com","city":"Bluffview","state":"MN","_is_real":true} +{"index":{"_id":"589"}} +{"account_number":589,"balance":33260,"firstname":"Ericka","lastname":"Cote","age":39,"gender":"F","address":"425 Bath Avenue","employer":"Venoflex","email":"erickacote@venoflex.com","city":"Blue","state":"CT","_is_real":true} +{"index":{"_id":"591"}} +{"account_number":591,"balance":48997,"firstname":"Rivers","lastname":"Macdonald","age":34,"gender":"F","address":"919 Johnson Street","employer":"Ziore","email":"riversmacdonald@ziore.com","city":"Townsend","state":"IL","_is_real":true} +{"index":{"_id":"596"}} +{"account_number":596,"balance":4063,"firstname":"Letitia","lastname":"Walker","age":26,"gender":"F","address":"963 Vanderveer Place","employer":"Zizzle","email":"letitiawalker@zizzle.com","city":"Rossmore","state":"ID","_is_real":true} +{"index":{"_id":"604"}} +{"account_number":604,"balance":10675,"firstname":"Isabel","lastname":"Gilliam","age":23,"gender":"M","address":"854 Broadway ","employer":"Zenthall","email":"isabelgilliam@zenthall.com","city":"Ventress","state":"WI","_is_real":true} +{"index":{"_id":"609"}} +{"account_number":609,"balance":28586,"firstname":"Montgomery","lastname":"Washington","age":30,"gender":"M","address":"169 Schroeders Avenue","employer":"Kongle","email":"montgomerywashington@kongle.com","city":"Croom","state":"AZ","_is_real":true} +{"index":{"_id":"611"}} +{"account_number":611,"balance":17528,"firstname":"Katherine","lastname":"Prince","age":33,"gender":"F","address":"705 Elm Avenue","employer":"Zillacon","email":"katherineprince@zillacon.com","city":"Rew","state":"MI","_is_real":true} +{"index":{"_id":"616"}} +{"account_number":616,"balance":25276,"firstname":"Jessie","lastname":"Mayer","age":35,"gender":"F","address":"683 Chester Avenue","employer":"Emtrak","email":"jessiemayer@emtrak.com","city":"Marysville","state":"HI","_is_real":true} +{"index":{"_id":"623"}} +{"account_number":623,"balance":20514,"firstname":"Rose","lastname":"Combs","age":32,"gender":"F","address":"312 Grimes Road","employer":"Aquamate","email":"rosecombs@aquamate.com","city":"Fostoria","state":"OH","_is_real":true} +{"index":{"_id":"628"}} +{"account_number":628,"balance":42736,"firstname":"Buckner","lastname":"Chen","age":37,"gender":"M","address":"863 Rugby Road","employer":"Jamnation","email":"bucknerchen@jamnation.com","city":"Camas","state":"TX","_is_real":true} +{"index":{"_id":"630"}} +{"account_number":630,"balance":46060,"firstname":"Leanne","lastname":"Jones","age":31,"gender":"M","address":"451 Bayview Avenue","employer":"Wazzu","email":"leannejones@wazzu.com","city":"Kylertown","state":"OK","_is_real":true} +{"index":{"_id":"635"}} +{"account_number":635,"balance":44705,"firstname":"Norman","lastname":"Gilmore","age":33,"gender":"M","address":"330 Gates Avenue","employer":"Comfirm","email":"normangilmore@comfirm.com","city":"Riceville","state":"TN","_is_real":true} +{"index":{"_id":"642"}} +{"account_number":642,"balance":32852,"firstname":"Reyna","lastname":"Harris","age":35,"gender":"M","address":"305 Powell Street","employer":"Bedlam","email":"reynaharris@bedlam.com","city":"Florence","state":"KS","_is_real":true} +{"index":{"_id":"647"}} +{"account_number":647,"balance":10147,"firstname":"Annabelle","lastname":"Velazquez","age":30,"gender":"M","address":"299 Kensington Walk","employer":"Sealoud","email":"annabellevelazquez@sealoud.com","city":"Soudan","state":"ME","_is_real":true} +{"index":{"_id":"654"}} +{"account_number":654,"balance":38695,"firstname":"Armstrong","lastname":"Frazier","age":25,"gender":"M","address":"899 Seeley Street","employer":"Zensor","email":"armstrongfrazier@zensor.com","city":"Cherokee","state":"UT","_is_real":true} +{"index":{"_id":"659"}} +{"account_number":659,"balance":29648,"firstname":"Dorsey","lastname":"Sosa","age":40,"gender":"M","address":"270 Aberdeen Street","employer":"Daycore","email":"dorseysosa@daycore.com","city":"Chamberino","state":"SC","_is_real":true} +{"index":{"_id":"661"}} +{"account_number":661,"balance":3679,"firstname":"Joanne","lastname":"Spencer","age":39,"gender":"F","address":"910 Montauk Avenue","employer":"Visalia","email":"joannespencer@visalia.com","city":"Valmy","state":"NH","_is_real":true} +{"index":{"_id":"666"}} +{"account_number":666,"balance":13880,"firstname":"Mcguire","lastname":"Lloyd","age":40,"gender":"F","address":"658 Just Court","employer":"Centrexin","email":"mcguirelloyd@centrexin.com","city":"Warren","state":"MT","_is_real":true} +{"index":{"_id":"673"}} +{"account_number":673,"balance":11303,"firstname":"Mcdaniel","lastname":"Harrell","age":33,"gender":"M","address":"565 Montgomery Place","employer":"Eyeris","email":"mcdanielharrell@eyeris.com","city":"Garnet","state":"NV","_is_real":true} +{"index":{"_id":"678"}} +{"account_number":678,"balance":43663,"firstname":"Ruby","lastname":"Shaffer","age":28,"gender":"M","address":"350 Clark Street","employer":"Comtrail","email":"rubyshaffer@comtrail.com","city":"Aurora","state":"MA","_is_real":true} +{"index":{"_id":"680"}} +{"account_number":680,"balance":31561,"firstname":"Melton","lastname":"Camacho","age":32,"gender":"F","address":"771 Montana Place","employer":"Insuresys","email":"meltoncamacho@insuresys.com","city":"Sparkill","state":"IN","_is_real":true} +{"index":{"_id":"685"}} +{"account_number":685,"balance":22249,"firstname":"Yesenia","lastname":"Rowland","age":24,"gender":"F","address":"193 Dekalb Avenue","employer":"Coriander","email":"yeseniarowland@coriander.com","city":"Lupton","state":"NC","_is_real":true} +{"index":{"_id":"692"}} +{"account_number":692,"balance":10435,"firstname":"Haney","lastname":"Barlow","age":21,"gender":"F","address":"267 Lenox Road","employer":"Egypto","email":"haneybarlow@egypto.com","city":"Detroit","state":"IN","_is_real":true} +{"index":{"_id":"697"}} +{"account_number":697,"balance":48745,"firstname":"Mallory","lastname":"Emerson","age":24,"gender":"F","address":"318 Dunne Court","employer":"Exoplode","email":"malloryemerson@exoplode.com","city":"Montura","state":"LA","_is_real":true} +{"index":{"_id":"700"}} +{"account_number":700,"balance":19164,"firstname":"Patel","lastname":"Durham","age":21,"gender":"F","address":"440 King Street","employer":"Icology","email":"pateldurham@icology.com","city":"Mammoth","state":"IL","_is_real":true} +{"index":{"_id":"705"}} +{"account_number":705,"balance":28415,"firstname":"Krystal","lastname":"Cross","age":22,"gender":"M","address":"604 Drew Street","employer":"Tubesys","email":"krystalcross@tubesys.com","city":"Dalton","state":"MO","_is_real":true} +{"index":{"_id":"712"}} +{"account_number":712,"balance":12459,"firstname":"Butler","lastname":"Alston","age":37,"gender":"M","address":"486 Hemlock Street","employer":"Quordate","email":"butleralston@quordate.com","city":"Verdi","state":"MS","_is_real":true} +{"index":{"_id":"717"}} +{"account_number":717,"balance":29270,"firstname":"Erickson","lastname":"Mcdonald","age":31,"gender":"M","address":"873 Franklin Street","employer":"Exotechno","email":"ericksonmcdonald@exotechno.com","city":"Jessie","state":"MS","_is_real":true} +{"index":{"_id":"724"}} +{"account_number":724,"balance":12548,"firstname":"Hopper","lastname":"Peck","age":31,"gender":"M","address":"849 Hendrickson Street","employer":"Uxmox","email":"hopperpeck@uxmox.com","city":"Faxon","state":"UT","_is_real":true} +{"index":{"_id":"729"}} +{"account_number":729,"balance":41812,"firstname":"Katy","lastname":"Rivera","age":36,"gender":"F","address":"791 Olive Street","employer":"Blurrybus","email":"katyrivera@blurrybus.com","city":"Innsbrook","state":"MI","_is_real":true} +{"index":{"_id":"731"}} +{"account_number":731,"balance":4994,"firstname":"Lorene","lastname":"Weiss","age":35,"gender":"M","address":"990 Ocean Court","employer":"Comvoy","email":"loreneweiss@comvoy.com","city":"Lavalette","state":"WI","_is_real":true} +{"index":{"_id":"736"}} +{"account_number":736,"balance":28677,"firstname":"Rogers","lastname":"Mcmahon","age":21,"gender":"F","address":"423 Cameron Court","employer":"Brainclip","email":"rogersmcmahon@brainclip.com","city":"Saddlebrooke","state":"FL","_is_real":true} +{"index":{"_id":"743"}} +{"account_number":743,"balance":14077,"firstname":"Susana","lastname":"Moody","age":23,"gender":"M","address":"842 Fountain Avenue","employer":"Bitrex","email":"susanamoody@bitrex.com","city":"Temperanceville","state":"TN","_is_real":true} +{"index":{"_id":"748"}} +{"account_number":748,"balance":38060,"firstname":"Ford","lastname":"Branch","age":25,"gender":"M","address":"926 Cypress Avenue","employer":"Buzzness","email":"fordbranch@buzzness.com","city":"Beason","state":"DC","_is_real":true} +{"index":{"_id":"750"}} +{"account_number":750,"balance":40481,"firstname":"Cherie","lastname":"Brooks","age":20,"gender":"F","address":"601 Woodhull Street","employer":"Kaggle","email":"cheriebrooks@kaggle.com","city":"Groton","state":"MA","_is_real":true} +{"index":{"_id":"755"}} +{"account_number":755,"balance":43878,"firstname":"Bartlett","lastname":"Conway","age":22,"gender":"M","address":"453 Times Placez","employer":"Konnect","email":"bartlettconway@konnect.com","city":"Belva","state":"VT","_is_real":true} +{"index":{"_id":"762"}} +{"account_number":762,"balance":10291,"firstname":"Amanda","lastname":"Head","age":20,"gender":"F","address":"990 Ocean Parkway","employer":"Zentury","email":"amandahead@zentury.com","city":"Hegins","state":"AR","_is_real":true} +{"index":{"_id":"767"}} +{"account_number":767,"balance":26220,"firstname":"Anthony","lastname":"Sutton","age":27,"gender":"F","address":"179 Fayette Street","employer":"Xiix","email":"anthonysutton@xiix.com","city":"Iberia","state":"TN","_is_real":true} +{"index":{"_id":"774"}} +{"account_number":774,"balance":35287,"firstname":"Lynnette","lastname":"Alvarez","age":38,"gender":"F","address":"991 Brightwater Avenue","employer":"Gink","email":"lynnettealvarez@gink.com","city":"Leola","state":"NC","_is_real":true} +{"index":{"_id":"779"}} +{"account_number":779,"balance":40983,"firstname":"Maggie","lastname":"Pace","age":32,"gender":"F","address":"104 Harbor Court","employer":"Bulljuice","email":"maggiepace@bulljuice.com","city":"Floris","state":"MA","_is_real":true} +{"index":{"_id":"781"}} +{"account_number":781,"balance":29961,"firstname":"Sanford","lastname":"Mullen","age":26,"gender":"F","address":"879 Dover Street","employer":"Zanity","email":"sanfordmullen@zanity.com","city":"Martinez","state":"TX","_is_real":true} +{"index":{"_id":"786"}} +{"account_number":786,"balance":3024,"firstname":"Rene","lastname":"Vang","age":33,"gender":"M","address":"506 Randolph Street","employer":"Isopop","email":"renevang@isopop.com","city":"Vienna","state":"NJ","_is_real":true} +{"index":{"_id":"793"}} +{"account_number":793,"balance":16911,"firstname":"Alford","lastname":"Compton","age":36,"gender":"M","address":"186 Veronica Place","employer":"Zyple","email":"alfordcompton@zyple.com","city":"Sugartown","state":"AK","_is_real":true} +{"index":{"_id":"798"}} +{"account_number":798,"balance":3165,"firstname":"Catherine","lastname":"Ward","age":30,"gender":"F","address":"325 Burnett Street","employer":"Dreamia","email":"catherineward@dreamia.com","city":"Glenbrook","state":"SD","_is_real":true} +{"index":{"_id":"801"}} +{"account_number":801,"balance":14954,"firstname":"Molly","lastname":"Maldonado","age":37,"gender":"M","address":"518 Maple Avenue","employer":"Straloy","email":"mollymaldonado@straloy.com","city":"Hebron","state":"WI","_is_real":true} +{"index":{"_id":"806"}} +{"account_number":806,"balance":36492,"firstname":"Carson","lastname":"Riddle","age":31,"gender":"M","address":"984 Lois Avenue","employer":"Terrago","email":"carsonriddle@terrago.com","city":"Leland","state":"MN","_is_real":true} +{"index":{"_id":"813"}} +{"account_number":813,"balance":30833,"firstname":"Ebony","lastname":"Bishop","age":20,"gender":"M","address":"487 Ridge Court","employer":"Optique","email":"ebonybishop@optique.com","city":"Fairmount","state":"WA","_is_real":true} +{"index":{"_id":"818"}} +{"account_number":818,"balance":24433,"firstname":"Espinoza","lastname":"Petersen","age":26,"gender":"M","address":"641 Glenwood Road","employer":"Futurity","email":"espinozapetersen@futurity.com","city":"Floriston","state":"MD","_is_real":true} +{"index":{"_id":"820"}} +{"account_number":820,"balance":1011,"firstname":"Shepard","lastname":"Ramsey","age":24,"gender":"F","address":"806 Village Court","employer":"Mantro","email":"shepardramsey@mantro.com","city":"Tibbie","state":"NV","_is_real":true} +{"index":{"_id":"825"}} +{"account_number":825,"balance":49000,"firstname":"Terra","lastname":"Witt","age":21,"gender":"F","address":"590 Conway Street","employer":"Insectus","email":"terrawitt@insectus.com","city":"Forbestown","state":"AR","_is_real":true} +{"index":{"_id":"832"}} +{"account_number":832,"balance":8582,"firstname":"Laura","lastname":"Gibbs","age":39,"gender":"F","address":"511 Osborn Street","employer":"Corepan","email":"lauragibbs@corepan.com","city":"Worcester","state":"KS","_is_real":true} +{"index":{"_id":"837"}} +{"account_number":837,"balance":14485,"firstname":"Amy","lastname":"Villarreal","age":35,"gender":"M","address":"381 Stillwell Place","employer":"Fleetmix","email":"amyvillarreal@fleetmix.com","city":"Sanford","state":"IA","_is_real":true} +{"index":{"_id":"844"}} +{"account_number":844,"balance":26840,"firstname":"Jill","lastname":"David","age":31,"gender":"M","address":"346 Legion Street","employer":"Zytrax","email":"jilldavid@zytrax.com","city":"Saticoy","state":"SC","_is_real":true} +{"index":{"_id":"849"}} +{"account_number":849,"balance":16200,"firstname":"Barry","lastname":"Chapman","age":26,"gender":"M","address":"931 Dekoven Court","employer":"Darwinium","email":"barrychapman@darwinium.com","city":"Whitestone","state":"WY","_is_real":true} +{"index":{"_id":"851"}} +{"account_number":851,"balance":22026,"firstname":"Henderson","lastname":"Price","age":33,"gender":"F","address":"530 Hausman Street","employer":"Plutorque","email":"hendersonprice@plutorque.com","city":"Brutus","state":"RI","_is_real":true} +{"index":{"_id":"856"}} +{"account_number":856,"balance":27583,"firstname":"Alissa","lastname":"Knox","age":25,"gender":"M","address":"258 Empire Boulevard","employer":"Geologix","email":"alissaknox@geologix.com","city":"Hartsville/Hartley","state":"MN","_is_real":true} +{"index":{"_id":"863"}} +{"account_number":863,"balance":23165,"firstname":"Melendez","lastname":"Fernandez","age":40,"gender":"M","address":"661 Johnson Avenue","employer":"Vixo","email":"melendezfernandez@vixo.com","city":"Farmers","state":"IL","_is_real":true} +{"index":{"_id":"868"}} +{"account_number":868,"balance":27624,"firstname":"Polly","lastname":"Barron","age":22,"gender":"M","address":"129 Frank Court","employer":"Geofarm","email":"pollybarron@geofarm.com","city":"Loyalhanna","state":"ND","_is_real":true} +{"index":{"_id":"870"}} +{"account_number":870,"balance":43882,"firstname":"Goff","lastname":"Phelps","age":21,"gender":"M","address":"164 Montague Street","employer":"Digigen","email":"goffphelps@digigen.com","city":"Weedville","state":"IL","_is_real":true} +{"index":{"_id":"875"}} +{"account_number":875,"balance":19655,"firstname":"Mercer","lastname":"Pratt","age":24,"gender":"M","address":"608 Perry Place","employer":"Twiggery","email":"mercerpratt@twiggery.com","city":"Eggertsville","state":"MO","_is_real":true} +{"index":{"_id":"882"}} +{"account_number":882,"balance":10895,"firstname":"Mari","lastname":"Landry","age":39,"gender":"M","address":"963 Gerald Court","employer":"Kenegy","email":"marilandry@kenegy.com","city":"Lithium","state":"NC","_is_real":true} +{"index":{"_id":"887"}} +{"account_number":887,"balance":31772,"firstname":"Eunice","lastname":"Watts","age":36,"gender":"F","address":"707 Stuyvesant Avenue","employer":"Memora","email":"eunicewatts@memora.com","city":"Westwood","state":"TN","_is_real":true} +{"index":{"_id":"894"}} +{"account_number":894,"balance":1031,"firstname":"Tyler","lastname":"Fitzgerald","age":32,"gender":"M","address":"787 Meserole Street","employer":"Jetsilk","email":"tylerfitzgerald@jetsilk.com","city":"Woodlands","state":"WV","_is_real":true} +{"index":{"_id":"899"}} +{"account_number":899,"balance":32953,"firstname":"Carney","lastname":"Callahan","age":23,"gender":"M","address":"724 Kimball Street","employer":"Mangelica","email":"carneycallahan@mangelica.com","city":"Tecolotito","state":"MT","_is_real":true} +{"index":{"_id":"902"}} +{"account_number":902,"balance":13345,"firstname":"Hallie","lastname":"Jarvis","age":23,"gender":"F","address":"237 Duryea Court","employer":"Anixang","email":"halliejarvis@anixang.com","city":"Boykin","state":"IN","_is_real":true} +{"index":{"_id":"907"}} +{"account_number":907,"balance":12961,"firstname":"Ingram","lastname":"William","age":36,"gender":"M","address":"826 Overbaugh Place","employer":"Genmex","email":"ingramwilliam@genmex.com","city":"Kimmell","state":"AK","_is_real":true} +{"index":{"_id":"914"}} +{"account_number":914,"balance":7120,"firstname":"Esther","lastname":"Bean","age":32,"gender":"F","address":"583 Macon Street","employer":"Applica","email":"estherbean@applica.com","city":"Homeworth","state":"MN","_is_real":true} +{"index":{"_id":"919"}} +{"account_number":919,"balance":39655,"firstname":"盛虹","lastname":"Hanson","age":27,"gender":"M","address":"557 Hart Place","employer":"Exospace","email":"shaunahanson@exospace.com","city":"Outlook","state":"LA","_is_real":true} +{"index":{"_id":"921"}} +{"account_number":921,"balance":49119,"firstname":"Barbara","lastname":"Wade","age":29,"gender":"M","address":"687 Hoyts Lane","employer":"Roughies","email":"barbarawade@roughies.com","city":"Sattley","state":"CO","_is_real":true} +{"index":{"_id":"926"}} +{"account_number":926,"balance":49433,"firstname":"Welch","lastname":"Mcgowan","age":21,"gender":"M","address":"833 Quincy Street","employer":"Atomica","email":"welchmcgowan@atomica.com","city":"Hampstead","state":"VT","_is_real":true} +{"index":{"_id":"933"}} +{"account_number":933,"balance":18071,"firstname":"Tabitha","lastname":"Cole","age":21,"gender":"F","address":"916 Rogers Avenue","employer":"Eclipto","email":"tabithacole@eclipto.com","city":"Lawrence","state":"TX","_is_real":true} +{"index":{"_id":"938"}} +{"account_number":938,"balance":9597,"firstname":"Sharron","lastname":"Santos","age":40,"gender":"F","address":"215 Matthews Place","employer":"Zenco","email":"sharronsantos@zenco.com","city":"Wattsville","state":"VT","_is_real":true} +{"index":{"_id":"940"}} +{"account_number":940,"balance":23285,"firstname":"Melinda","lastname":"Mendoza","age":38,"gender":"M","address":"806 Kossuth Place","employer":"Kneedles","email":"melindamendoza@kneedles.com","city":"Coaldale","state":"OK","_is_real":true} +{"index":{"_id":"945"}} +{"account_number":945,"balance":23085,"firstname":"Hansen","lastname":"Hebert","age":33,"gender":"F","address":"287 Conduit Boulevard","employer":"Capscreen","email":"hansenhebert@capscreen.com","city":"Taycheedah","state":"AK","_is_real":true} +{"index":{"_id":"952"}} +{"account_number":952,"balance":21430,"firstname":"Angelique","lastname":"Weeks","age":33,"gender":"M","address":"659 Reeve Place","employer":"Exodoc","email":"angeliqueweeks@exodoc.com","city":"Turpin","state":"MD","_is_real":true} +{"index":{"_id":"957"}} +{"account_number":957,"balance":11373,"firstname":"Michael","lastname":"Giles","age":31,"gender":"M","address":"668 Court Square","employer":"Yogasm","email":"michaelgiles@yogasm.com","city":"Rosburg","state":"WV","_is_real":true} +{"index":{"_id":"964"}} +{"account_number":964,"balance":26154,"firstname":"Elena","lastname":"Waller","age":34,"gender":"F","address":"618 Crystal Street","employer":"Insurety","email":"elenawaller@insurety.com","city":"Gallina","state":"NY","_is_real":true} +{"index":{"_id":"969"}} +{"account_number":969,"balance":22214,"firstname":"Briggs","lastname":"Lynn","age":30,"gender":"M","address":"952 Lester Court","employer":"Quinex","email":"briggslynn@quinex.com","city":"Roland","state":"ID","_is_real":true} +{"index":{"_id":"971"}} +{"account_number":971,"balance":22772,"firstname":"Gabrielle","lastname":"Reilly","age":32,"gender":"F","address":"964 Tudor Terrace","employer":"Blanet","email":"gabriellereilly@blanet.com","city":"Falmouth","state":"AL","_is_real":true} +{"index":{"_id":"976"}} +{"account_number":976,"balance":31707,"firstname":"Mullen","lastname":"Tanner","age":26,"gender":"M","address":"711 Whitney Avenue","employer":"Pulze","email":"mullentanner@pulze.com","city":"Mooresburg","state":"MA","_is_real":true} +{"index":{"_id":"983"}} +{"account_number":983,"balance":47205,"firstname":"Mattie","lastname":"Eaton","age":24,"gender":"F","address":"418 Allen Avenue","employer":"Trasola","email":"mattieeaton@trasola.com","city":"Dupuyer","state":"NJ","_is_real":true} +{"index":{"_id":"988"}} +{"account_number":988,"balance":17803,"firstname":"Lucy","lastname":"Castro","age":34,"gender":"F","address":"425 Fleet Walk","employer":"Geekfarm","email":"lucycastro@geekfarm.com","city":"Mulino","state":"VA","_is_real":true} +{"index":{"_id":"990"}} +{"account_number":990,"balance":44456,"firstname":"Kelly","lastname":"Steele","age":35,"gender":"M","address":"809 Hoyt Street","employer":"Eschoir","email":"kellysteele@eschoir.com","city":"Stewartville","state":"ID","_is_real":true} +{"index":{"_id":"995"}} +{"account_number":995,"balance":21153,"firstname":"Phelps","lastname":"Parrish","age":25,"gender":"M","address":"666 Miller Place","employer":"Pearlessa","email":"phelpsparrish@pearlessa.com","city":"Brecon","state":"ME","_is_real":true} +{"index":{"_id":"9001"}} +{"account_number":9001,"balance":1000,"firstname":"Fake","lastname":"Alpha","age":25,"gender":"M","address":"1 Fake St","city":"Faketown","state":"CA","_is_real":false} +{"index":{"_id":"9002"}} +{"account_number":9002,"balance":2000,"firstname":"Fake","lastname":"Beta","age":30,"gender":"F","address":"2 Fake Ave","city":"Fakeville","state":"NY","_is_real":false} +{"index":{"_id":"9003"}} +{"account_number":9003,"balance":3000,"firstname":"Fake","lastname":"Gamma","age":35,"gender":"M","address":"3 Fake Blvd","city":"Fakeburg","state":"TX","_is_real":false} diff --git a/integ-test/src/test/resources/bank_extended.json b/integ-test/src/test/resources/bank_extended.json new file mode 100644 index 00000000000..c62744996c7 --- /dev/null +++ b/integ-test/src/test/resources/bank_extended.json @@ -0,0 +1,20 @@ +{"index":{"_id":"1"}} +{"account_number":1,"balance":39225,"firstname":"Amber JOHnny","lastname":"Duke Willmington","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL","male":true,"birthdate":"2017-10-23","_is_real":true} +{"index":{"_id":"6"}} +{"account_number":6,"balance":5686,"firstname":"Hattie","lastname":"Bond","age":36,"gender":"M","address":"671 Bristol Street","employer":"Netagy","email":"hattiebond@netagy.com","city":"Dante","state":"TN","male":true,"birthdate":"2017-11-20","_is_real":true} +{"index":{"_id":"13"}} +{"account_number":13,"balance":32838,"firstname":"Nanette","lastname":"Bates","age":28,"gender":"F","address":"789 Madison Street","employer":"Quility","email":"nanettebates@quility.com","city":"Nogal","state":"VA","male":false,"birthdate":"2018-06-23","_is_real":true} +{"index":{"_id":"18"}} +{"account_number":18,"balance":4180,"firstname":"Dale","lastname":"Adams","age":33,"gender":"M","address":"467 Hutchinson Court","employer":"Boink","email":"daleadams@boink.com","city":"Orick","state":"MD","male":true,"birthdate":1542152000000,"_is_real":true} +{"index":{"_id":"20"}} +{"account_number":20,"balance":16418,"firstname":"Elinor","lastname":"Ratliff","age":36,"gender":"M","address":"282 Kings Place","employer":"Scentric","email":"elinorratliff@scentric.com","city":"Ribera","state":"WA","male":true,"birthdate":"2018-06-27","_is_real":true} +{"index":{"_id":"25"}} +{"account_number":25,"balance":40540,"firstname":"Virginia","lastname":"Ayala","age":39,"gender":"F","address":"171 Putnam Avenue","employer":"Filodyne","email":"virginiaayala@filodyne.com","city":"Nicholson","state":"PA","male":false,"birthdate":"2018-08-19","_is_real":true} +{"index":{"_id":"32"}} +{"account_number":32,"balance":48086,"firstname":"Dillard","lastname":"Mcpherson","age":34,"gender":"F","address":"702 Quentin Street","employer":"Quailcom","email":"dillardmcpherson@quailcom.com","city":"Veguita","state":"IN","male":false,"birthdate":"2018-08-11","_is_real":true} +{"index":{"_id":"99"}} +{"account_number":99,"balance":1000,"firstname":"Fake","lastname":"One","age":25,"gender":"M","address":"1 Fake Street","employer":"Fakecorp","email":"fakeone@fakecorp.com","city":"Faketown","state":"CA","male":true,"birthdate":"2020-01-01","_is_real":false} +{"index":{"_id":"100"}} +{"account_number":100,"balance":2000,"firstname":"Fake","lastname":"Two","age":30,"gender":"F","address":"2 Fake Avenue","employer":"Fakecorp","email":"faketwo@fakecorp.com","city":"Fakeville","state":"NY","male":false,"birthdate":"2020-06-15","_is_real":false} +{"index":{"_id":"101"}} +{"account_number":101,"balance":3000,"firstname":"Fake","lastname":"Three","age":35,"gender":"M","address":"3 Fake Boulevard","employer":"Fakecorp","email":"fakethree@fakecorp.com","city":"Fakeburg","state":"TX","male":true,"birthdate":"2021-03-10","_is_real":false} diff --git a/integ-test/src/test/resources/datetime_simple.json b/integ-test/src/test/resources/datetime_simple.json new file mode 100644 index 00000000000..7ed96384a24 --- /dev/null +++ b/integ-test/src/test/resources/datetime_simple.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"date_value": "2020-10-13 13:00:00"} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml index d0a2f80d866..903980bed98 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_transpose.yaml @@ -3,20 +3,21 @@ calcite: LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) LogicalProject(column_names=[$0], row 1=[$1], row 2=[$2], row 3=[$3], row 4=[$4]) LogicalAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0) FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5]) - LogicalProject(_value_transpose_=[CAST($19):VARCHAR NOT NULL], $f20=[TRIM(FLAG(BOTH), ' ', $18)], $f21=[=($17, 1)], $f22=[=($17, 2)], $f23=[=($17, 3)], $f24=[=($17, 4)]) + LogicalProject(_value_transpose_=[CAST($19):VARCHAR NOT NULL], $f20=[CAST(TRIM(FLAG(BOTH), ' ', $18)):VARCHAR NOT NULL], $f21=[=($17, 1)], $f22=[=($17, 2)], $f23=[=($17, 3)], $f24=[=($17, 4)]) LogicalFilter(condition=[IS NOT NULL($19)]) - LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_transpose_=[$17], column_names=[$18], _value_transpose_=[CASE(=($18, 'account_number'), CAST($0):VARCHAR NOT NULL, =($18, 'firstname'), CAST($1):VARCHAR NOT NULL, =($18, 'address'), CAST($2):VARCHAR NOT NULL, =($18, 'balance'), CAST($3):VARCHAR NOT NULL, =($18, 'gender'), CAST($4):VARCHAR NOT NULL, =($18, 'city'), CAST($5):VARCHAR NOT NULL, =($18, 'employer'), CAST($6):VARCHAR NOT NULL, =($18, 'state'), CAST($7):VARCHAR NOT NULL, =($18, 'age'), CAST($8):VARCHAR NOT NULL, =($18, 'email'), CAST($9):VARCHAR NOT NULL, =($18, 'lastname'), CAST($10):VARCHAR NOT NULL, null:NULL)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_transpose_=[$17], column_names=[$18], _value_transpose_=[CASE(=($18, 'account_number'), CAST($0):VARCHAR NOT NULL, =($18, 'firstname '), CAST($1):VARCHAR NOT NULL, =($18, 'address '), CAST($2):VARCHAR NOT NULL, =($18, 'balance '), CAST($3):VARCHAR NOT NULL, =($18, 'gender '), CAST($4):VARCHAR NOT NULL, =($18, 'city '), CAST($5):VARCHAR NOT NULL, =($18, 'employer '), CAST($6):VARCHAR NOT NULL, =($18, 'state '), CAST($7):VARCHAR NOT NULL, =($18, 'age '), CAST($8):VARCHAR NOT NULL, =($18, 'email '), CAST($9):VARCHAR NOT NULL, =($18, 'lastname '), CAST($10):VARCHAR NOT NULL, null:NULL)]) LogicalJoin(condition=[true], joinType=[inner]) LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], _row_number_transpose_=[ROW_NUMBER() OVER ()]) LogicalSort(fetch=[5]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) - LogicalValues(tuples=[[{ 'account_number' }, { 'firstname' }, { 'address' }, { 'balance' }, { 'gender' }, { 'city' }, { 'employer' }, { 'state' }, { 'age' }, { 'email' }, { 'lastname' }]]) + LogicalValues(tuples=[[{ 'account_number' }, { 'firstname ' }, { 'address ' }, { 'balance ' }, { 'gender ' }, { 'city ' }, { 'employer ' }, { 'state ' }, { 'age ' }, { 'email ' }, { 'lastname ' }]]) physical: | EnumerableLimit(fetch=[10000]) EnumerableAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0) FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5]) - EnumerableCalc(expr#0..12=[{inputs}], expr#13=['account_number'], expr#14=[=($t12, $t13)], expr#15=[CAST($t0):VARCHAR NOT NULL], expr#16=['firstname'], expr#17=[=($t12, $t16)], expr#18=[CAST($t1):VARCHAR NOT NULL], expr#19=['address'], expr#20=[=($t12, $t19)], expr#21=[CAST($t2):VARCHAR NOT NULL], expr#22=['balance'], expr#23=[=($t12, $t22)], expr#24=[CAST($t3):VARCHAR NOT NULL], expr#25=['gender'], expr#26=[=($t12, $t25)], expr#27=[CAST($t4):VARCHAR NOT NULL], expr#28=['city'], expr#29=[=($t12, $t28)], expr#30=[CAST($t5):VARCHAR NOT NULL], expr#31=['employer'], expr#32=[=($t12, $t31)], expr#33=[CAST($t6):VARCHAR NOT NULL], expr#34=['state'], expr#35=[=($t12, $t34)], expr#36=[CAST($t7):VARCHAR NOT NULL], expr#37=['age'], expr#38=[=($t12, $t37)], expr#39=[CAST($t8):VARCHAR NOT NULL], expr#40=['email'], expr#41=[=($t12, $t40)], expr#42=[CAST($t9):VARCHAR NOT NULL], expr#43=['lastname'], expr#44=[=($t12, $t43)], expr#45=[CAST($t10):VARCHAR NOT NULL], expr#46=[null:NULL], expr#47=[CASE($t14, $t15, $t17, $t18, $t20, $t21, $t23, $t24, $t26, $t27, $t29, $t30, $t32, $t33, $t35, $t36, $t38, $t39, $t41, $t42, $t44, $t45, $t46)], expr#48=[CAST($t47):VARCHAR NOT NULL], expr#49=[FLAG(BOTH)], expr#50=[' '], expr#51=[TRIM($t49, $t50, $t12)], expr#52=[1], expr#53=[=($t11, $t52)], expr#54=[2], expr#55=[=($t11, $t54)], expr#56=[3], expr#57=[=($t11, $t56)], expr#58=[4], expr#59=[=($t11, $t58)], _value_transpose_=[$t48], $f20=[$t51], $f21=[$t53], $f22=[$t55], $f23=[$t57], $f24=[$t59]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=['account_number'], expr#14=[=($t12, $t13)], expr#15=[CAST($t0):VARCHAR NOT NULL], expr#16=['firstname '], expr#17=[=($t12, $t16)], expr#18=[CAST($t1):VARCHAR NOT NULL], expr#19=['address '], expr#20=[=($t12, $t19)], expr#21=[CAST($t2):VARCHAR NOT NULL], expr#22=['balance '], expr#23=[=($t12, $t22)], expr#24=[CAST($t3):VARCHAR NOT NULL], expr#25=['gender '], expr#26=[=($t12, $t25)], expr#27=[CAST($t4):VARCHAR NOT NULL], expr#28=['city '], expr#29=[=($t12, $t28)], expr#30=[CAST($t5):VARCHAR NOT NULL], expr#31=['employer '], expr#32=[=($t12, $t31)], expr#33=[CAST($t6):VARCHAR NOT NULL], expr#34=['state '], expr#35=[=($t12, $t34)], expr#36=[CAST($t7):VARCHAR NOT NULL], expr#37=['age '], expr#38=[=($t12, $t37)], expr#39=[CAST($t8):VARCHAR NOT NULL], expr#40=['email '], expr#41=[=($t12, $t40)], expr#42=[CAST($t9):VARCHAR NOT NULL], expr#43=['lastname '], expr#44=[=($t12, $t43)], expr#45=[CAST($t10):VARCHAR NOT NULL], expr#46=[null:NULL], expr#47=[CASE($t14, $t15, $t17, $t18, $t20, $t21, $t23, $t24, $t26, $t27, $t29, $t30, $t32, $t33, $t35, $t36, $t38, $t39, $t41, $t42, $t44, $t45, $t46)], expr#48=[CAST($t47):VARCHAR NOT NULL], expr#49=[FLAG(BOTH)], expr#50=[' '], expr#51=[TRIM($t49, $t50, $t12)], expr#52=[CAST($t51):VARCHAR NOT NULL], expr#53=[1], expr#54=[=($t11, $t53)], expr#55=[2], expr#56=[=($t11, $t55)], expr#57=[3], expr#58=[=($t11, $t57)], expr#59=[4], expr#60=[=($t11, $t59)], _value_transpose_=[$t48], $f20=[$t52], $f21=[$t54], $f22=[$t56], $f23=[$t58], $f24=[$t60]) EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) EnumerableWindow(window#0=[window(rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) - EnumerableCalc(expr#0=[{inputs}], expr#1=[Sarg['account_number', 'address':CHAR(14), 'age':CHAR(14), 'balance':CHAR(14), 'city':CHAR(14), 'email':CHAR(14), 'employer':CHAR(14), 'firstname':CHAR(14), 'gender':CHAR(14), 'lastname':CHAR(14), 'state':CHAR(14)]:CHAR(14)], expr#2=[SEARCH($t0, $t1)], column_names=[$t0], $condition=[$t2]) - EnumerableValues(tuples=[[{ 'account_number' }, { 'firstname' }, { 'address' }, { 'balance' }, { 'gender' }, { 'city' }, { 'employer' }, { 'state' }, { 'age' }, { 'email' }, { 'lastname' }]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[Sarg['account_number', 'address ', 'age ', 'balance ', 'city ', 'email ', 'employer ', 'firstname ', 'gender ', 'lastname ', 'state ']:CHAR(14)], expr#2=[SEARCH($t0, $t1)], column_names=[$t0], $condition=[$t2]) + EnumerableValues(tuples=[[{ 'account_number' }, { 'firstname ' }, { 'address ' }, { 'balance ' }, { 'gender ' }, { 'city ' }, { 'employer ' }, { 'state ' }, { 'age ' }, { 'email ' }, { 'lastname ' }]]) + diff --git a/integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json new file mode 100644 index 00000000000..67b26a7c8f7 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/account_extended_index_mapping.json @@ -0,0 +1,53 @@ +{ + "mappings": { + "properties": { + "gender": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "address": { + "type": "text", + "fielddata": true + }, + "firstname": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "_is_real": { + "type": "boolean" + }, + "lastname": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "state": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json new file mode 100644 index 00000000000..2512751d9d2 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/bank_extended_index_mapping.json @@ -0,0 +1,61 @@ +{ + "mappings": { + "properties": { + "account_number": { + "type": "long" + }, + "address": { + "type": "text" + }, + "age": { + "type": "integer" + }, + "balance": { + "type": "long" + }, + "birthdate": { + "type": "date" + }, + "city": { + "type": "keyword" + }, + "email": { + "type": "text" + }, + "employer": { + "type": "text" + }, + "firstname": { + "type": "keyword" + }, + "gender": { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "_is_real": { + "type": "boolean" + }, + "lastname": { + "type": "keyword" + }, + "male": { + "type": "boolean" + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json new file mode 100644 index 00000000000..fa2283f68f7 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/datetime_simple_index_mapping.json @@ -0,0 +1,10 @@ +{ + "mappings": { + "properties": { + "date_value": { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" + } + } + } +} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml new file mode 100644 index 00000000000..4207a6c3005 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5481.yml @@ -0,0 +1,69 @@ +# Issue: https://github.com/opensearch-project/sql/issues/5481 +# A timestamp range comparison AND'd with an IN clause on another field must push down and +# return rows. Calcite folds the IN into a Sarg and strips the timestamp literal's UDT; without +# the field-type-keyed fix the range query ships an unformatted date and the shard rejects it. +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + + - do: + indices.create: + index: issue5481 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + event_time: + type: date + severity: + type: keyword + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "issue5481", "_id": "1"}}' + - '{"event_time": "2026-05-28T10:00:00Z", "severity": "ERROR"}' + - '{"index": {"_index": "issue5481", "_id": "2"}}' + - '{"event_time": "2026-05-28T10:05:00Z", "severity": "WARN"}' + - '{"index": {"_index": "issue5481", "_id": "3"}}' + - '{"event_time": "2026-05-28T10:10:00Z", "severity": "INFO"}' + - '{"index": {"_index": "issue5481", "_id": "4"}}' + - '{"event_time": "2026-05-28T10:15:00Z", "severity": "ERROR"}' + - '{"index": {"_index": "issue5481", "_id": "5"}}' + - '{"event_time": "2026-05-28T10:20:00Z", "severity": "WARN"}' + - '{"index": {"_index": "issue5481", "_id": "6"}}' + - '{"event_time": "2026-05-28T10:25:00Z", "severity": "DEBUG"}' + +--- +teardown: + - do: + indices.delete: + index: issue5481 + ignore_unavailable: true + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: false + +--- +"Issue 5481: timestamp range AND keyword IN pushes down and returns rows": + - skip: + features: + - headers + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=issue5481 | where event_time > timestamp('2026-05-28 10:08:00') | where severity in ('ERROR', 'WARN') | fields severity, event_time | sort event_time + + - match: { total: 2 } + - match: { schema: [ { name: severity, type: "string" }, { name: event_time, type: "timestamp" } ] } + - match: { datarows: [ [ "ERROR", "2026-05-28 10:15:00" ], [ "WARN", "2026-05-28 10:20:00" ] ] } diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java index ba28ee83252..f2e0c9e1fd4 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/format/ErrorMessageFactory.java @@ -6,6 +6,7 @@ package org.opensearch.sql.legacy.executor.format; import org.opensearch.OpenSearchException; +import org.opensearch.sql.common.error.ErrorReport; public class ErrorMessageFactory { /** @@ -25,6 +26,10 @@ public static ErrorMessage createErrorMessage(Exception e, int status) { OpenSearchException exception = (OpenSearchException) unwrapCause(e); return new OpenSearchErrorMessage(exception, exception.status().getStatus()); } + // Unwrap ErrorReport so the error type reflects the underlying cause, not the wrapper. + if (e instanceof ErrorReport) { + return new ErrorMessage(((ErrorReport) e).getCause(), status); + } return new ErrorMessage(e, status); } diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java index 31baaced852..0984f8e1f45 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/ErrorMessageFactoryTest.java @@ -9,6 +9,7 @@ import org.junit.Test; import org.opensearch.OpenSearchException; import org.opensearch.core.rest.RestStatus; +import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.legacy.executor.format.ErrorMessage; import org.opensearch.sql.legacy.executor.format.ErrorMessageFactory; import org.opensearch.sql.legacy.executor.format.OpenSearchErrorMessage; @@ -42,6 +43,25 @@ public void nonOpenSearchExceptionWithWrappedEsExceptionCauseShouldCreateEsError Assert.assertTrue(msg instanceof OpenSearchErrorMessage); } + @Test + public void errorReportShouldRenderUnderlyingCauseType() { + Exception exception = + ErrorReport.wrap(new IllegalArgumentException("Field [x] not found.")).build(); + ErrorMessage msg = + ErrorMessageFactory.createErrorMessage(exception, RestStatus.BAD_REQUEST.getStatus()); + Assert.assertFalse(msg instanceof OpenSearchErrorMessage); + Assert.assertTrue(msg.toString().contains("IllegalArgumentException")); + Assert.assertFalse(msg.toString().contains("ErrorReport")); + } + + @Test + public void errorReportWrappingEsExceptionShouldCreateEsErrorMessage() { + Exception exception = ErrorReport.wrap(new OpenSearchException(nonOpenSearchThrowable)).build(); + ErrorMessage msg = + ErrorMessageFactory.createErrorMessage(exception, RestStatus.NOT_FOUND.getStatus()); + Assert.assertTrue(msg instanceof OpenSearchErrorMessage); + } + @Test public void nonOpenSearchExceptionWithMultiLayerWrappedEsExceptionCauseShouldCreateEsErrorMessage() { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 79d49a143de..76de0c30a08 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -14,8 +14,12 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import org.apache.commons.lang3.EnumUtils; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; /** The extension of ExprType in OpenSearch. */ @EqualsAndHashCode @@ -297,7 +301,26 @@ private static void validateAliasType(Map result) { (key, value) -> { if (value instanceof OpenSearchAliasType && value.getOriginalPath().isPresent()) { String originalPath = value.getOriginalPath().get(); - result.put(key, new OpenSearchAliasType(originalPath, result.get(originalPath))); + OpenSearchDataType target = result.get(originalPath); + if (target == null) { + throw ErrorReport.wrap( + new SemanticCheckException( + String.format( + "Alias field [%s] refers to unresolved path [%s].", + key, originalPath))) + .code(ErrorCode.FIELD_NOT_FOUND) + .stage(QueryProcessingStage.ANALYZING) + .location("while resolving alias fields in the index mapping") + .context("alias_field", key) + .context("alias_path", originalPath) + .suggestion( + "The alias path must point to an existing field in the mapping; a text" + + " multi-field (e.g. \"" + + originalPath + + ".keyword\") or a removed/renamed field is not a valid alias target.") + .build(); + } + result.put(key, new OpenSearchAliasType(originalPath, target)); } }); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java index d37957d4a9f..648888ce9b7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/DedupPushdownRule.java @@ -17,6 +17,7 @@ import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.rules.SubstitutionRule; import org.apache.calcite.rex.RexInputRef; @@ -47,6 +48,24 @@ protected DedupPushdownRule(Config config) { @Override protected void onMatchImpl(RelOptRuleCall call) { final LogicalDedup logicalDedup = call.rel(0); + if (call.rels[1] instanceof LogicalFilter filter) { + // Push the filter into the scan, then synthesize an identity project so the standard + // apply() can run on the resulting Dedup → Project → Scan shape. If the filter is only + // partially pushable, pushDownFilter returns a Filter wrapping the residual condition over + // the new scan; we can't strip a residual filter without breaking semantics, so bail. + final CalciteLogicalIndexScan scan = call.rel(2); + if (!(scan.pushDownFilter(filter) instanceof CalciteLogicalIndexScan newScan)) { + return; + } + RelBuilder relBuilder = call.builder(); + relBuilder.push(newScan); + // force=true so the identity project is materialized (apply() requires a LogicalProject) + relBuilder.project( + relBuilder.fields(), newScan.getRowType().getFieldNames(), /* force= */ true); + LogicalProject identityProject = (LogicalProject) relBuilder.build(); + apply(call, logicalDedup, identityProject, newScan); + return; + } final LogicalProject projectWithExpr = call.rel(1); final CalciteLogicalIndexScan scan = call.rel(2); apply(call, logicalDedup, projectWithExpr, scan); @@ -226,6 +245,27 @@ public interface Config extends OpenSearchRuleConfig { .predicate(Config::tableScanChecker) .noInputs()))); + // +- LogicalDedup + // +- LogicalFilter (e.g. a `where` predicate preserved below dedup by + // +- CalciteLogicalIndexScan PPLSimplifyDedupRule when an upstream `where` is folded + // into the bucket-non-null filter) + Config WITH_FILTER = + ImmutableDedupPushdownRule.Config.builder() + .build() + .withDescription("Dedup-to-Aggregate-WithFilter") + .withOperandSupplier( + b0 -> + b0.operand(LogicalDedup.class) + .predicate(dedup -> !dedup.getKeepEmpty()) + .oneInput( + b1 -> + b1.operand(LogicalFilter.class) + .oneInput( + b2 -> + b2.operand(CalciteLogicalIndexScan.class) + .predicate(Config::tableScanChecker) + .noInputs()))); + /** * Project must be not pushed since the name of expression would lose after project pushed. E.g. * in query "eval new_a = a + 1 | dedup b", the "new_a" will lose. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index 0068f445ce7..3c8508cc455 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -50,6 +50,8 @@ public class OpenSearchIndexRules { SortIndexScanRule.Config.DEFAULT.toRule(); private static final DedupPushdownRule DEDUP_PUSH_DOWN = DedupPushdownRule.Config.DEFAULT.toRule(); + private static final DedupPushdownRule DEDUP_PUSH_DOWN_WITH_FILTER = + DedupPushdownRule.Config.WITH_FILTER.toRule(); private static final SortProjectExprTransposeRule SORT_PROJECT_EXPR_TRANSPOSE = SortProjectExprTransposeRule.Config.DEFAULT.toRule(); private static final ExpandCollationOnProjectExprRule EXPAND_COLLATION_ON_PROJECT_EXPR = @@ -75,6 +77,7 @@ public class OpenSearchIndexRules { LIMIT_INDEX_SCAN, SORT_INDEX_SCAN, DEDUP_PUSH_DOWN, + DEDUP_PUSH_DOWN_WITH_FILTER, SORT_PROJECT_EXPR_TRANSPOSE, SORT_AGGREGATION_METRICS_RULE, RARE_TOP_PUSH_DOWN, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 6389fb28395..67a31a7f56b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -784,16 +784,29 @@ private QueryExpression like(RexCall call) { private static QueryExpression constructQueryExpressionForSearch( RexCall call, SwapResult pair) { + boolean isTimeStamp = + (pair.getKey() instanceof NamedFieldExpression namedField) + && namedField.isTimeStampType(); if (isSearchWithComplementedPoints(call)) { + // A NOT IN over a timestamp field cannot use a flat terms query: timestamp values must be + // normalized + formatted (see in()/termsQuery, which does neither). Decompose into the + // complement of an OR of formatted per-point range queries, mirroring the range branch. + if (isTimeStamp) { + return pointsAsTimestampOr(call, pair).not(); + } return QueryExpression.create(pair.getKey()).notIn(pair.getValue()); } else if (isSearchWithPoints(call)) { + // IN over a timestamp field: the flat terms query in in() emits unformatted values like + // '2018-06-23 00:00:00' that the date field cannot parse. Decompose into an OR of formatted + // per-point range queries (equals(point, true) builds gte/lte + format="date_time"), the + // same path the range branch below and visitCompare's `=` already take. + if (isTimeStamp) { + return pointsAsTimestampOr(call, pair); + } return QueryExpression.create(pair.getKey()).in(pair.getValue()); } else { Sarg sarg = pair.getValue().literal.getValueAs(Sarg.class); Set> rangeSet = requireNonNull(sarg).rangeSet.asRanges(); - boolean isTimeStamp = - (pair.getKey() instanceof NamedFieldExpression namedField) - && namedField.isTimeStampType(); List queryExpressions = rangeSet.stream() .map( @@ -811,6 +824,30 @@ private static QueryExpression constructQueryExpressionForSearch( } } + /** + * Builds an OR of per-point {@code equals(point, true)} query expressions for a points-only + * Sarg over a timestamp field. Each point becomes a {@code gte/lte} range query with {@code + * format="date_time"}, so the timestamp value is normalized to the date field's accepted format + * — unlike a flat terms query, which emits unformatted values the date field rejects. + */ + private static QueryExpression pointsAsTimestampOr(RexCall call, SwapResult pair) { + RexLiteral literal = (RexLiteral) call.getOperands().get(1); + Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); + Set> ranges = + (sarg.isComplementedPoints() ? sarg.negate() : sarg).rangeSet.asRanges(); + List queryExpressions = + ranges.stream() + .map( + range -> + QueryExpression.create(pair.getKey()) + .equals(sargPointValue(range.lowerEndpoint()), true)) + .toList(); + if (queryExpressions.size() == 1) { + return queryExpressions.getFirst(); + } + return CompoundQueryExpression.or(queryExpressions.toArray(new QueryExpression[0])); + } + private QueryExpression andOr(RexCall call) { QueryExpression[] expressions = new QueryExpression[call.getOperands().size()]; PredicateAnalyzerException firstError = null; @@ -1378,10 +1415,12 @@ public QueryExpression notLike(LiteralExpression literal) { @Override public QueryExpression equals(LiteralExpression literal) { - Object value = literal.value(); - if (literal.isDateTime()) { + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + if (isTimeStamp) { builder = - addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value).lte(value)); + addFormatIfNecessary( + isTimeStamp, rangeQuery(getFieldReference()).gte(value).lte(value)); } else { builder = termQuery(getFieldReferenceForTermQuery(), value); } @@ -1390,12 +1429,15 @@ public QueryExpression equals(LiteralExpression literal) { @Override public QueryExpression notEquals(LiteralExpression literal) { - Object value = literal.value(); - if (literal.isDateTime()) { + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + if (isTimeStamp) { builder = boolQuery() - .should(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gt(value))) - .should(addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lt(value))); + .should( + addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).gt(value))) + .should( + addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).lt(value))); } else { builder = boolQuery() @@ -1408,32 +1450,48 @@ public QueryExpression notEquals(LiteralExpression literal) { @Override public QueryExpression gt(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gt(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).gt(value)); return this; } @Override public QueryExpression gte(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).gte(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).gte(value)); return this; } @Override public QueryExpression lt(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lt(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).lt(value)); return this; } @Override public QueryExpression lte(LiteralExpression literal) { - Object value = literal.value(); - builder = addFormatIfNecessary(literal, rangeQuery(getFieldReference()).lte(value)); + boolean isTimeStamp = isFieldOrLiteralDateTime(literal); + Object value = convertEndpointValue(literal.value(), isTimeStamp); + builder = addFormatIfNecessary(isTimeStamp, rangeQuery(getFieldReference()).lte(value)); return this; } + /** + * Whether the comparison is a timestamp/date range. The field type is the reliable signal: + * {@code literal.isDateTime()} reads the literal's UDT, which {@link + * org.apache.calcite.rex.RexSimplify} can strip (to VARCHAR) when a sibling clause is folded + * into a {@code Sarg}, e.g. {@code @timestamp > X AND severityText IN (...)}. Falling back to + * {@code rel.isTimeStampType()} keeps ISO-8601 normalization and the {@code "date_time"} format + * hint on the range query. + */ + private boolean isFieldOrLiteralDateTime(LiteralExpression literal) { + return literal.isDateTime() || (rel != null && rel.isTimeStampType()); + } + @Override public QueryExpression match(String query, Map optionalArguments) { builder = new MatchQuery().build(getFieldReference(), query, optionalArguments); @@ -1580,6 +1638,11 @@ public QueryExpression between(Range range, boolean isTimeStamp) { } private Object convertEndpointValue(Object value, boolean isTimeStamp) { + // Shared normalization entry point: guard a null endpoint so the timestamp branch's + // value.toString() cannot NPE. sargPointValue never produces null from a non-null input. + if (value == null) { + return null; + } value = sargPointValue(value); return isTimeStamp ? timestampValueForPushDown(value.toString()) : value; } @@ -1712,16 +1775,19 @@ public static ScriptSortBuilder.ScriptSortType getScriptSortType(RelDataType rel } /** - * By default, range queries on date/time need use the format of the source to parse the literal. - * So we need to specify that the literal has "date_time" format + * Range queries on date/time fields need the source format to parse the literal, so we attach the + * {@code "date_time"} format. The caller resolves whether the comparison is a timestamp range + * from the field type (see {@link SimpleQueryExpression#isFieldOrLiteralDateTime}) rather than + * the literal's UDT, which {@link org.apache.calcite.rex.RexSimplify} can strip when a sibling + * clause is folded into a {@code Sarg}. * - * @param literal literal value - * @param rangeQueryBuilder query builder to optionally add {@code format} expression - * @return existing builder with possible {@code format} attribute + * @param isTimeStamp whether the comparison endpoint is a timestamp/date range endpoint + * @param rangeQueryBuilder query builder to optionally add the {@code format} attribute + * @return the same builder, with {@code format("date_time")} added when {@code isTimeStamp} */ private static RangeQueryBuilder addFormatIfNecessary( - LiteralExpression literal, RangeQueryBuilder rangeQueryBuilder) { - if (literal.isDateTime()) { + boolean isTimeStamp, RangeQueryBuilder rangeQueryBuilder) { + if (isTimeStamp) { rangeQueryBuilder.format("date_time"); } return rangeQueryBuilder; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 40985130c52..1479ccfb615 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -41,8 +41,12 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.common.error.ErrorCode; +import org.opensearch.sql.common.error.ErrorReport; +import org.opensearch.sql.common.error.QueryProcessingStage; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchDataTypeTest { @@ -483,6 +487,53 @@ public void test_AliasType() { () -> assertEquals("original_path2", aliasTypeOnDouble.getOriginalPath().orElseThrow())); } + @Test + public void traverseAndFlatten_alias_to_unresolvable_path_throws_descriptive_error() { + // Alias path targets a text multi-field, which is not in the flattened mapping. + Map keywordAliasTree = + Map.of( + "source", + textKeywordType, + "source_alias", + new OpenSearchAliasType("source.keyword", OpenSearchDataType.of(MappingType.Invalid))); + ErrorReport keywordError = + assertThrows( + ErrorReport.class, () -> OpenSearchDataType.traverseAndFlatten(keywordAliasTree)); + assertAll( + () -> assertEquals(ErrorCode.FIELD_NOT_FOUND, keywordError.getCode()), + () -> assertEquals(QueryProcessingStage.ANALYZING, keywordError.getStage()), + () -> assertTrue(keywordError.getCause() instanceof SemanticCheckException), + () -> + assertEquals( + "Alias field [source_alias] refers to unresolved path [source.keyword].", + keywordError.getCause().getMessage()), + () -> assertEquals("source_alias", keywordError.getContext().get("alias_field")), + () -> assertEquals("source.keyword", keywordError.getContext().get("alias_path")), + () -> + assertTrue( + keywordError.getSuggestion().contains("\"source.keyword.keyword\"") + && keywordError.getSuggestion().contains("not a valid alias target"))); + + // Alias path targets a field that does not exist. + Map missingFieldTree = + Map.of( + "col1", + textType, + "col_alias", + new OpenSearchAliasType("missing", OpenSearchDataType.of(MappingType.Invalid))); + ErrorReport missingError = + assertThrows( + ErrorReport.class, () -> OpenSearchDataType.traverseAndFlatten(missingFieldTree)); + assertAll( + () -> assertEquals(ErrorCode.FIELD_NOT_FOUND, missingError.getCode()), + () -> assertTrue(missingError.getCause() instanceof SemanticCheckException), + () -> + assertEquals( + "Alias field [col_alias] refers to unresolved path [missing].", + missingError.getCause().getMessage()), + () -> assertEquals("missing", missingError.getContext().get("alias_path"))); + } + @Test public void test_parseMapping_on_AliasType() { Map indexMapping1 = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 3deb39d58c0..041063f62a0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -1147,6 +1147,138 @@ void notEquals_generatesBoolQueryForDateTime() throws ExpressionNotAnalyzableExc result.toString()); } + /** + * RexSimplify can strip the EXPR_TIMESTAMP UDT off a literal when a sibling clause is folded into + * a Sarg (e.g. {@code @timestamp > X AND severityText IN (...)}), leaving the literal as plain + * VARCHAR. The comparison must still emit a {@code format("date_time")} range query keyed off the + * field's type so the shard's default date parser accepts the value. + */ + @Test + void gt_normalizesVarcharLiteralAgainstTimestampField() throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.GREATER_THAN, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : false, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, + result.toString()); + } + + // Companion stripped-VARCHAR-literal tests for the remaining range shapes (equals -> gte+lte, + // notEquals -> two-should bool, lte -> single range). Each must produce the same DSL as its + // intact-UDT counterpart, proving the field-type fallback in isFieldOrLiteralDateTime keeps + // ISO-8601 normalization + format("date_time") on every comparison op, not just gt. See #5481. + @Test + void equals_normalizesVarcharLiteralAgainstTimestampField() + throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.EQUALS, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, + result.toString()); + } + + @Test + void notEquals_normalizesVarcharLiteralAgainstTimestampField() + throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.NOT_EQUALS, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(BoolQueryBuilder.class, result); + assertEquals( + """ + { + "bool" : { + "should" : [ + { + "range" : { + "d" : { + "from" : "1987-02-03T04:34:56.000Z", + "to" : null, + "include_lower" : false, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }, + { + "range" : { + "d" : { + "from" : null, + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : false, + "format" : "date_time", + "boost" : 1.0 + } + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }\ + """, + result.toString()); + } + + @Test + void lte_normalizesVarcharLiteralAgainstTimestampField() throws ExpressionNotAnalyzableException { + RexLiteral varcharLiteral = (RexLiteral) builder.makeLiteral("1987-02-03 04:34:56"); + RexNode call = builder.makeCall(SqlStdOperatorTable.LESS_THAN_OR_EQUAL, field4, varcharLiteral); + QueryBuilder result = PredicateAnalyzer.analyze(call, schema, fieldTypes); + + assertInstanceOf(RangeQueryBuilder.class, result); + assertEquals( + """ + { + "range" : { + "d" : { + "from" : null, + "to" : "1987-02-03T04:34:56.000Z", + "include_lower" : true, + "include_upper" : true, + "format" : "date_time", + "boost" : 1.0 + } + } + }\ + """, + result.toString()); + } + @Test void gte_generatesRangeQueryWithFormatForDateTime() throws ExpressionNotAnalyzableException { RexNode call = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index 0eee03102bb..ab8b923e3ae 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -63,6 +63,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.sql.ast.statement.ExplainMode; import org.opensearch.sql.common.response.ResponseListener; +import org.opensearch.sql.common.utils.QueryContext; import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelper; import org.opensearch.sql.datasources.auth.DataSourceUserAuthorizationHelperImpl; @@ -241,6 +242,7 @@ private BiFunction createSqlAnalyticsRout || !unifiedQueryHandler.isAnalyticsIndex(sqlRequest.getQuery(), QueryType.SQL)) { return false; } + LOGGER.info("[{}] Routing SQL query to analytics engine", QueryContext.getRequestId()); if (sqlRequest.isExplainRequest()) { unifiedQueryHandler.explain( sqlRequest.getQuery(), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 844513ed836..d81ce7b5137 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -13,10 +13,12 @@ import java.util.Map; import java.util.Optional; import org.apache.calcite.rel.RelNode; +import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.ThreadContext; import org.opensearch.analytics.exec.QueryPlanExecutor; +import org.opensearch.analytics.exec.profile.QueryProfile; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.action.ActionListener; @@ -39,6 +41,7 @@ import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import org.opensearch.sql.utils.SystemIndexUtils; import org.opensearch.transport.client.node.NodeClient; /** @@ -94,7 +97,17 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - return true; + // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline + try (UnifiedQueryContext context = buildParsingContext(queryType)) { + boolean systemCatalog = + extractIndexName(query, queryType, context) + .map(RestUnifiedQueryAction::isSystemCatalog) + .orElse(false); + return !systemCatalog; + } catch (Exception e) { + // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. + return !isLegacySystemCatalogQuery(query); + } } try (UnifiedQueryContext context = buildParsingContext(queryType)) { return extractIndexName(query, queryType, context) @@ -106,6 +119,16 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { } } + private static boolean isSystemCatalog(String name) { + return SystemIndexUtils.isSystemIndex(name) + || SystemIndexUtils.DATASOURCES_TABLE_NAME.equals(name); + } + + private static boolean isLegacySystemCatalogQuery(String query) { + String trimmed = query.trim(); + return Strings.CI.startsWith(trimmed, "SHOW ") || Strings.CI.startsWith(trimmed, "DESCRIBE "); + } + private String stripSchemaPrefix(String indexName) { int lastDot = indexName.lastIndexOf('.'); return lastDot >= 0 ? indexName.substring(lastDot + 1) : indexName; @@ -137,7 +160,10 @@ public void execute( // schema we plan against and the state the executor uses are the same view. org.opensearch.analytics.QueryRequestContext queryCtx = contextProvider.getContext(); - UnifiedQueryContext context = buildContext(queryType, profiling, queryCtx); + // Disable SQL-layer phase profiling when analytics engine profiling is active. + // Our QueryProfile (stages, tasks, timing) is strictly more detailed and replaces + // it. + UnifiedQueryContext context = buildContext(queryType, false, queryCtx); ActionListener closingListener = wrapWithContextClose(context, listener); try { @@ -145,11 +171,19 @@ public void execute( RelNode plan = planner.plan(query); CalcitePlanContext planContext = context.getPlanContext(); plan = addQuerySizeLimit(plan, planContext); - analyticsEngine.execute( - plan, - planContext, - queryCtx, - createQueryListener(queryType, closingListener)); + if (profiling) { + analyticsEngine.executeWithProfile( + plan, + planContext, + queryCtx, + createQueryListener(queryType, closingListener)); + } else { + analyticsEngine.execute( + plan, + planContext, + queryCtx, + createQueryListener(queryType, closingListener)); + } } catch (Exception e) { closingListener.onFailure(e); } @@ -270,6 +304,10 @@ public void onResponse(QueryResponse response) { formatter.format( new QueryResult( response.getSchema(), response.getResults(), response.getCursor(), langSpec)); + if (response.getProfile() != null) { + // Append profile and error (if any) to the JSON response + result = appendProfileToJson(result, response.getProfile(), response.getError()); + } transportListener.onResponse(new TransportPPLQueryResponse(result)); } @@ -280,6 +318,32 @@ public void onFailure(Exception e) { }; } + private static String appendProfileToJson(String json, QueryProfile profile, Throwable error) { + try { + StringBuilder extra = new StringBuilder(); + // Append profile + org.opensearch.core.xcontent.XContentBuilder builder = + org.opensearch.common.xcontent.XContentFactory.jsonBuilder(); + profile.toXContent(builder, org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS); + extra.append(",\"profile\":").append(builder.toString()); + // Append error if query partially failed + if (error != null) { + extra + .append(",\"error\":{\"type\":\"") + .append(error.getClass().getSimpleName()) + .append("\",\"reason\":\"") + .append(error.getMessage() != null ? error.getMessage().replace("\"", "\\\"") : "") + .append("\"}"); + } + if (json.endsWith("}")) { + return json.substring(0, json.length() - 1) + extra + "}"; + } + return json; + } catch (Exception e) { + return json; + } + } + private static Runnable withCurrentContext(final Runnable task) { final Map currentContext = ThreadContext.getImmutableContext(); return () -> { diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index b7c2d2c9e11..fd231c29076 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -14,6 +14,8 @@ import java.util.Optional; import java.util.function.Supplier; import org.apache.calcite.rel.RelNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; @@ -59,6 +61,8 @@ public class TransportPPLQueryAction extends HandledTransportAction { + private static final Logger LOG = LogManager.getLogger(TransportPPLQueryAction.class); + private final Injector injector; private final Supplier pplEnabled; @@ -171,6 +175,7 @@ protected void doExecute( // Route to analytics engine for non-Lucene (e.g., Parquet-backed) indices. if (unifiedQueryHandler != null && unifiedQueryHandler.isAnalyticsIndex(transformedRequest.getRequest(), QueryType.PPL)) { + LOG.info("[{}] Routing PPL query to analytics engine", QueryContext.getRequestId()); if (transformedRequest.isExplainRequest()) { unifiedQueryHandler.explain( transformedRequest.getRequest(), diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 84b75161bd0..111597bb587 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -21,6 +21,7 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; +import org.opensearch.indices.IndicesService; import org.opensearch.sql.executor.QueryType; import org.opensearch.transport.client.node.NodeClient; @@ -142,7 +143,7 @@ public void nullAndEmptyQueriesRouteToLucene() { } @Test - public void showStatementRoutesToLucene() { + public void showStatementNotRoutedToAnalyticsEngine() { registerIndex( "parquet_logs", Settings.builder() @@ -154,7 +155,7 @@ public void showStatementRoutesToLucene() { } @Test - public void describeStatementRoutesToLucene() { + public void describeStatementNotRoutedToAnalyticsEngine() { registerIndex( "parquet_logs", Settings.builder() @@ -165,6 +166,79 @@ public void describeStatementRoutesToLucene() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } + @Test + public void showStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("SHOW TABLES LIKE 'parquet_logs'", QueryType.SQL)); + } + + @Test + public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); + } + + @Test + public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + assertTrue(action.isAnalyticsIndex("SELECT * FROM parquet_logs", QueryType.SQL)); + } + + @Test + public void unparseableQueryRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + // malformed -> AE re-parses & reports + assertTrue(action.isAnalyticsIndex("SELECT FROM WHERE", QueryType.SQL)); + } + + @Test + public void legacyShowNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + // unquoted LIKE is rejected by the V2 parser, but still belongs on the default pipeline + assertFalse(action.isAnalyticsIndex("SHOW TABLES LIKE %", QueryType.SQL)); + } + + @Test + public void legacyDescribeNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + // legacy DESCRIBE syntax is rejected by the V2 parser, but belongs on the default pipeline + assertFalse(action.isAnalyticsIndex("DESCRIBE my_index", QueryType.SQL)); + } + + @Test + public void pplDescribeNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("describe parquet_logs", QueryType.PPL)); + } + + @Test + public void pplShowDatasourcesNotRoutedToAnalyticsEngineUnderClusterComposite() { + enableClusterComposite(); + assertFalse(action.isAnalyticsIndex("show datasources", QueryType.PPL)); + } + + @Test + public void pplDataQueryStillRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + assertTrue(action.isAnalyticsIndex("source = parquet_logs | fields ts", QueryType.PPL)); + } + + @Test + public void pplUnparseableQueryRoutesToAnalyticsUnderClusterComposite() { + enableClusterComposite(); + // malformed -> AE re-parses & reports + assertTrue(action.isAnalyticsIndex("source = parquet_logs | | fields ts", QueryType.PPL)); + } + + private void enableClusterComposite() { + when(clusterService.getSettings()) + .thenReturn( + Settings.builder() + .put( + IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.getKey(), "composite") + .build()); + } + private void registerIndex(String name, Settings settings) { IndexMetadata indexMetadata = mock(IndexMetadata.class); when(indexMetadata.getSettings()).thenReturn(settings); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java index ab07cd9b5c1..ec3b57d02cc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java @@ -110,6 +110,20 @@ public RelNode getRelNode(String ppl) { return root; } + /** + * Get the root RelNode of the given PPL query without merging adjacent filters. Useful for + * regression tests that need to exercise rule ordering against the un-merged shape that PPL + * actually emits to the production HEP optimizer. + */ + public RelNode getRelNodeRaw(String ppl) { + CalcitePlanContext context = createBuilderContext(); + Query query = (Query) plan(pplParser, ppl); + planTransformer.analyze(query.getPlan(), context); + RelNode root = context.relBuilder.build(); + System.out.println(root.explain()); + return root; + } + private RelNode mergeAdjacentFilters(RelNode relNode) { HepProgram program = new HepProgramBuilder().addRuleInstance(FilterMergeRule.Config.DEFAULT.toRule()).build(); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java index ca1a789b0f4..5f32c1b85bb 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLDedupTest.java @@ -5,9 +5,15 @@ package org.opensearch.sql.ppl.calcite; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.plan.hep.HepProgramBuilder; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rules.FilterMergeRule; import org.apache.calcite.test.CalciteAssert; +import org.junit.Assert; import org.junit.Test; +import org.opensearch.sql.calcite.plan.rule.PPLSimplifyDedupRule; public class CalcitePPLDedupTest extends CalcitePPLAbstractTest { @@ -353,4 +359,129 @@ public void testSortFieldProjectedAwayBeforeDedup() { + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); } + + /** + * When a user {@code where} precedes {@code dedup}, the user filter sits adjacent to the + * bucket-non-null filter that PPL emits for the dedup pattern. {@link PPLSimplifyDedupRule} must + * fold the dedup pattern into a {@link org.opensearch.sql.calcite.plan.rel.LogicalDedup} so that + * {@code DedupPushdownRule} can match it; otherwise dedup falls through to the in-memory {@code + * ROW_NUMBER} window form, defeating dedup pushdown to the shard. The user predicate must be + * preserved as a separate filter below the new {@code LogicalDedup}. + */ + @Test + public void testDedupAfterWhereProducesLogicalDedup() { + String ppl = "source=EMP | where SAL > 1000 | dedup DEPTNO"; + RelNode raw = getRelNodeRaw(ppl); + + // Sanity: the un-merged plan has both the bucket-non-null filter and the user where filter + // adjacent to each other above the scan — exactly the shape that triggered the original bug. + String rawExplain = raw.explain(); + Assert.assertTrue( + "Raw plan should contain the bucket-non-null filter:\n" + rawExplain, + rawExplain.contains("IS NOT NULL")); + Assert.assertTrue( + "Raw plan should contain the user where filter:\n" + rawExplain, + rawExplain.contains(">($5, 1000)")); + Assert.assertTrue( + "Raw plan should contain ROW_NUMBER prior to simplification:\n" + rawExplain, + rawExplain.contains("ROW_NUMBER")); + + // Apply rules in the order PPLSimplifyDedupRule -> FilterMergeRule. + HepProgram program = + new HepProgramBuilder() + .addRuleInstance(PPLSimplifyDedupRule.DEDUP_SIMPLIFY_RULE) + .addRuleInstance(FilterMergeRule.Config.DEFAULT.toRule()) + .build(); + RelNode optimized = runHepPlanner(raw, program); + + String optimizedExplain = optimized.explain(); + Assert.assertTrue( + "Optimized plan should contain LogicalDedup so DedupPushdownRule can match it:\n" + + optimizedExplain, + optimizedExplain.contains("LogicalDedup")); + Assert.assertFalse( + "Optimized plan should not retain ROW_NUMBER after simplification:\n" + optimizedExplain, + optimizedExplain.contains("ROW_NUMBER")); + Assert.assertTrue( + "User where predicate must be preserved as a filter below LogicalDedup:\n" + + optimizedExplain, + optimizedExplain.contains(">($5, 1000)")); + } + + /** + * Companion to {@link #testDedupAfterWhereProducesLogicalDedup} that pins the user-visible + * contract: with a {@code where} preceding {@code dedup}, a {@code LogicalDedup} must be produced + * regardless of the order in which {@link FilterMergeRule} and {@link PPLSimplifyDedupRule} fire. + * + *

    The simplify rule's bucket-non-null operand predicate is order-independent — it accepts both + * a pure {@code IS NOT NULL} and an {@code AND} that contains an {@code IS NOT NULL} on a + * partition key — so {@code FilterMergeRule} firing first no longer disables dedup pushdown. This + * guards against re-introducing the original regression by reordering, removing, or adding rules + * to {@code CalciteToolsHelper#HEP_PROGRAM}. + */ + @Test + public void testDedupAfterWhereProducesLogicalDedupRegardlessOfRuleOrder() { + String ppl = "source=EMP | where SAL > 1000 | dedup DEPTNO"; + RelNode raw = getRelNodeRaw(ppl); + + HepProgram program = + new HepProgramBuilder() + .addRuleInstance(FilterMergeRule.Config.DEFAULT.toRule()) + .addRuleInstance(PPLSimplifyDedupRule.DEDUP_SIMPLIFY_RULE) + .build(); + RelNode optimized = runHepPlanner(raw, program); + + String optimizedExplain = optimized.explain(); + Assert.assertTrue( + "Even with FilterMergeRule firing first, PPLSimplifyDedupRule must still produce" + + " LogicalDedup so DedupPushdownRule can match it:\n" + + optimizedExplain, + optimizedExplain.contains("LogicalDedup")); + Assert.assertFalse( + "ROW_NUMBER window form should be removed after simplification:\n" + optimizedExplain, + optimizedExplain.contains("ROW_NUMBER")); + Assert.assertTrue( + "User where predicate must be preserved as a filter below LogicalDedup, even when" + + " FilterMergeRule fired first and folded it into the bucket-non-null filter:\n" + + optimizedExplain, + optimizedExplain.contains(">($5, 1000)")); + } + + /** + * Mirrors the exact shape of {@code CalciteToolsHelper.HEP_PROGRAM} used in production ({@code + * addRuleCollection(List.of(FilterMergeRule, PPLSimplifyDedupRule))}). The two sequenced-{@code + * addRuleInstance} tests above prove the rule fires under either ordering, but production runs + * both rules in a single collection instruction. This test pins that exact shape to catch any + * addRuleCollection-vs-addRuleInstance traversal differences. + */ + @Test + public void testDedupAfterWhereProducesLogicalDedupWithProductionHepProgram() { + String ppl = "source=EMP | where SAL > 1000 | dedup DEPTNO"; + RelNode raw = getRelNodeRaw(ppl); + + HepProgram program = + new HepProgramBuilder() + .addRuleCollection( + java.util.List.of( + FilterMergeRule.Config.DEFAULT.toRule(), + PPLSimplifyDedupRule.DEDUP_SIMPLIFY_RULE)) + .build(); + RelNode optimized = runHepPlanner(raw, program); + + String optimizedExplain = optimized.explain(); + Assert.assertTrue( + "Production HEP_PROGRAM (addRuleCollection) must still produce LogicalDedup:\n" + + optimizedExplain, + optimizedExplain.contains("LogicalDedup")); + Assert.assertFalse( + "ROW_NUMBER window form should be removed under production HEP_PROGRAM:\n" + + optimizedExplain, + optimizedExplain.contains("ROW_NUMBER")); + } + + private static RelNode runHepPlanner(RelNode root, HepProgram program) { + HepPlanner planner = new HepPlanner(program); + planner.setRoot(root); + return planner.findBestExp(); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java index 69bc1ae2638..a9be30cb147 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLTransposeTest.java @@ -6,7 +6,10 @@ package org.opensearch.sql.ppl.calcite; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.test.CalciteAssert; +import org.junit.Assert; import org.junit.Test; public class CalcitePPLTransposeTest extends CalcitePPLAbstractTest { @@ -25,8 +28,8 @@ public void testSimpleCountWithTranspose() { + " FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5], row" + " 5_null=[MAX($0) FILTER $6])\n" + " LogicalProject(_value_transpose_=[CAST($3):VARCHAR NOT NULL]," - + " $f4=[TRIM(FLAG(BOTH), ' '," - + " $2)], $f5=[=($1, 1)], $f6=[=($1, 2)], $f7=[=($1, 3)], $f8=[=($1, 4)], $f9=[=($1," + + " $f4=[CAST(TRIM(FLAG(BOTH), ' ', $2)):VARCHAR NOT NULL], $f5=[=($1, 1)]," + + " $f6=[=($1, 2)], $f7=[=($1, 3)], $f8=[=($1, 4)], $f9=[=($1," + " 5)])\n" + " LogicalFilter(condition=[IS NOT NULL($3)])\n" + " LogicalProject(c=[$0], _row_number_transpose_=[$1], column=[$2]," @@ -41,16 +44,13 @@ public void testSimpleCountWithTranspose() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column`) `column`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" - + " `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 2) `row 2`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" + "SELECT CAST(TRIM(`column`) AS STRING) `column`, MAX(CAST(`_value_transpose_` AS STRING))" + + " FILTER (WHERE `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_`" + + " AS STRING)) FILTER (WHERE `_row_number_transpose_` = 2) `row 2`," + + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE `_row_number_transpose_` =" + + " 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" + " `_row_number_transpose_` = 4) `row 4`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 5) `row 5`\n" + + " FILTER (WHERE `_row_number_transpose_` = 5) `row 5`\n" + "FROM (SELECT `t0`.`c`, `t0`.`_row_number_transpose_`, `t1`.`column`, CASE WHEN" + " `t1`.`column` = 'c' THEN CAST(`t0`.`c` AS STRING) ELSE NULL END" + " `_value_transpose_`\n" @@ -58,7 +58,7 @@ public void testSimpleCountWithTranspose() { + "FROM `scott`.`EMP`) `t0`\n" + "CROSS JOIN (VALUES ('c')) `t1` (`column`)) `t2`\n" + "WHERE `t2`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column`)"; + + "GROUP BY CAST(TRIM(`column`) AS STRING)"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -75,14 +75,14 @@ public void testMultipleAggregatesWithAliasesTranspose() { + " FILTER $3], row 3_null=[MAX($0) FILTER $4], row 4_null=[MAX($0) FILTER $5], row" + " 5_null=[MAX($0) FILTER $6])\n" + " LogicalProject(_value_transpose_=[CAST($6):VARCHAR NOT NULL]," - + " $f7=[TRIM(FLAG(BOTH), ' '," - + " $5)], $f8=[=($4, 1)], $f9=[=($4, 2)], $f10=[=($4, 3)], $f11=[=($4, 4)]," + + " $f7=[CAST(TRIM(FLAG(BOTH), ' ', $5)):VARCHAR NOT NULL], $f8=[=($4, 1)]," + + " $f9=[=($4, 2)], $f10=[=($4, 3)], $f11=[=($4, 4)]," + " $f12=[=($4, 5)])\n" + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + " LogicalProject(avg_sal=[$0], max_sal=[$1], min_sal=[$2], cnt=[$3]," + " _row_number_transpose_=[$4], column=[$5], _value_transpose_=[CASE(=($5, 'avg_sal')," + " NUMBER_TO_STRING($0), =($5, 'max_sal'), NUMBER_TO_STRING($1), =($5, 'min_sal')," - + " NUMBER_TO_STRING($2), =($5, 'cnt'), CAST($3):VARCHAR NOT NULL, null:NULL)])\n" + + " NUMBER_TO_STRING($2), =($5, 'cnt '), CAST($3):VARCHAR NOT NULL, null:NULL)])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalProject(avg_sal=[$0], max_sal=[$1], min_sal=[$2], cnt=[$3]," + " _row_number_transpose_=[ROW_NUMBER() OVER ()])\n" @@ -91,7 +91,7 @@ public void testMultipleAggregatesWithAliasesTranspose() { + " LogicalProject(SAL=[$5])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" + " LogicalValues(tuples=[[{ 'avg_sal' }, { 'max_sal' }, { 'min_sal' }, {" - + " 'cnt' }]])\n"; + + " 'cnt ' }]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "column=avg_sal; row 1=2073.214285; row 2=null; row 3=null; row 4=null; row 5=null\n" @@ -102,31 +102,28 @@ public void testMultipleAggregatesWithAliasesTranspose() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column`) `column`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" - + " `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 2) `row 2`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" + "SELECT CAST(TRIM(`column`) AS STRING) `column`, MAX(CAST(`_value_transpose_` AS STRING))" + + " FILTER (WHERE `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_`" + + " AS STRING)) FILTER (WHERE `_row_number_transpose_` = 2) `row 2`," + + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE `_row_number_transpose_` =" + + " 3) `row 3`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" + " `_row_number_transpose_` = 4) `row 4`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 5) `row 5`\n" + + " FILTER (WHERE `_row_number_transpose_` = 5) `row 5`\n" + "FROM (SELECT `t1`.`avg_sal`, `t1`.`max_sal`, `t1`.`min_sal`, `t1`.`cnt`," + " `t1`.`_row_number_transpose_`, `t2`.`column`, CASE WHEN `t2`.`column` = 'avg_sal'" + " THEN NUMBER_TO_STRING(`t1`.`avg_sal`) WHEN `t2`.`column` = 'max_sal' THEN" + " NUMBER_TO_STRING(`t1`.`max_sal`) WHEN `t2`.`column` = 'min_sal' THEN" - + " NUMBER_TO_STRING(`t1`.`min_sal`) WHEN `t2`.`column` = 'cnt' THEN CAST(`t1`.`cnt` AS" - + " STRING) ELSE NULL END `_value_transpose_`\n" + + " NUMBER_TO_STRING(`t1`.`min_sal`) WHEN `t2`.`column` = 'cnt ' THEN" + + " CAST(`t1`.`cnt` AS STRING) ELSE NULL END `_value_transpose_`\n" + "FROM (SELECT AVG(`SAL`) `avg_sal`, MAX(`SAL`) `max_sal`, MIN(`SAL`) `min_sal`," + " COUNT(*) `cnt`, ROW_NUMBER() OVER () `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t1`\n" + "CROSS JOIN (VALUES ('avg_sal'),\n" + "('max_sal'),\n" + "('min_sal'),\n" - + "('cnt')) `t2` (`column`)) `t3`\n" + + "('cnt ')) `t2` (`column`)) `t3`\n" + "WHERE `t3`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column`)"; + + "GROUP BY CAST(TRIM(`column`) AS STRING)"; /* "SELECT `column`, MAX(CASE WHEN `__row_id__` = 1 THEN CAST(`value` AS STRING) ELSE NULL" @@ -164,18 +161,18 @@ public void testTransposeWithLimit() { + " LogicalAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0)" + " FILTER $3], row 3_null=[MAX($0) FILTER $4])\n" + " LogicalProject(_value_transpose_=[CAST($6):VARCHAR NOT NULL]," - + " $f7=[TRIM(FLAG(BOTH), ' '," - + " $5)], $f8=[=($4, 1)], $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + + " $f7=[CAST(TRIM(FLAG(BOTH), ' ', $5)):VARCHAR NOT NULL], $f8=[=($4, 1)]," + + " $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + " LogicalProject(ENAME=[$0], COMM=[$1], JOB=[$2], SAL=[$3]," + " _row_number_transpose_=[$4], column=[$5], _value_transpose_=[CASE(=($5, 'ENAME')," - + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM'), NUMBER_TO_STRING($1), =($5, 'JOB')," - + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL'), NUMBER_TO_STRING($3), null:NULL)])\n" + + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM '), NUMBER_TO_STRING($1), =($5, 'JOB ')," + + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL '), NUMBER_TO_STRING($3), null:NULL)])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalProject(ENAME=[$1], COMM=[$6], JOB=[$2], SAL=[$5]," + " _row_number_transpose_=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM' }, { 'JOB' }, { 'SAL'" + + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM ' }, { 'JOB ' }, { 'SAL '" + " }]])\n"; verifyLogical(root, expectedLogical); @@ -188,31 +185,39 @@ public void testTransposeWithLimit() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column`) `column`, MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" - + " `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 2) `row 2`, MAX(CAST(`_value_transpose_` AS STRING))" - + " FILTER (WHERE" - + " `_row_number_transpose_` = 3) `row 3`\n" + "SELECT CAST(TRIM(`column`) AS STRING) `column`, MAX(CAST(`_value_transpose_` AS STRING))" + + " FILTER (WHERE `_row_number_transpose_` = 1) `row 1`, MAX(CAST(`_value_transpose_`" + + " AS STRING)) FILTER (WHERE `_row_number_transpose_` = 2) `row 2`," + + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE `_row_number_transpose_` =" + + " 3) `row 3`\n" + "FROM (SELECT `t`.`ENAME`, `t`.`COMM`, `t`.`JOB`, `t`.`SAL`," + " `t`.`_row_number_transpose_`, `t0`.`column`, CASE WHEN `t0`.`column` = 'ENAME' THEN" - + " CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column` = 'COMM' THEN" - + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column` = 'JOB' THEN CAST(`t`.`JOB` AS" - + " STRING) WHEN `t0`.`column` = 'SAL' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE NULL END" + + " CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column` = 'COMM ' THEN" + + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column` = 'JOB ' THEN CAST(`t`.`JOB` AS" + + " STRING) WHEN `t0`.`column` = 'SAL ' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE NULL END" + " `_value_transpose_`\n" + "FROM (SELECT `ENAME`, `COMM`, `JOB`, `SAL`, ROW_NUMBER() OVER ()" + " `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t`\n" + "CROSS JOIN (VALUES ('ENAME'),\n" - + "('COMM'),\n" - + "('JOB'),\n" - + "('SAL')) `t0` (`column`)) `t1`\n" + + "('COMM '),\n" + + "('JOB '),\n" + + "('SAL ')) `t0` (`column`)) `t1`\n" + "WHERE `t1`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column`)"; + + "GROUP BY CAST(TRIM(`column`) AS STRING)"; verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testTransposeColumnAxisUsesUnboundedVarchar() { + RelNode root = getRelNode("source=EMP | fields ENAME, COMM, JOB, SAL | transpose 3"); + RelDataType columnType = root.getRowType().getFieldList().get(0).getType(); + + Assert.assertEquals(SqlTypeName.VARCHAR, columnType.getSqlTypeName()); + Assert.assertEquals(RelDataType.PRECISION_NOT_SPECIFIED, columnType.getPrecision()); + } + @Test public void testTransposeWithValueFieldNameCollision() { // Reproduce issue #5172: hardcoded 'value' unpivot column collides with @@ -237,19 +242,19 @@ public void testTransposeWithLimitColumnName() { + " LogicalAggregate(group=[{1}], row 1_null=[MAX($0) FILTER $2], row 2_null=[MAX($0)" + " FILTER $3], row 3_null=[MAX($0) FILTER $4])\n" + " LogicalProject(_value_transpose_=[CAST($6):VARCHAR NOT NULL]," - + " $f7=[TRIM(FLAG(BOTH), ' '," - + " $5)], $f8=[=($4, 1)], $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + + " $f7=[CAST(TRIM(FLAG(BOTH), ' ', $5)):VARCHAR NOT NULL], $f8=[=($4, 1)]," + + " $f9=[=($4, 2)], $f10=[=($4, 3)])\n" + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + " LogicalProject(ENAME=[$0], COMM=[$1], JOB=[$2], SAL=[$3]," + " _row_number_transpose_=[$4], column_names=[$5]," + " _value_transpose_=[CASE(=($5, 'ENAME')," - + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM'), NUMBER_TO_STRING($1), =($5, 'JOB')," - + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL'), NUMBER_TO_STRING($3), null:NULL)])\n" + + " CAST($0):VARCHAR NOT NULL, =($5, 'COMM '), NUMBER_TO_STRING($1), =($5, 'JOB ')," + + " CAST($2):VARCHAR NOT NULL, =($5, 'SAL '), NUMBER_TO_STRING($3), null:NULL)])\n" + " LogicalJoin(condition=[true], joinType=[inner])\n" + " LogicalProject(ENAME=[$1], COMM=[$6], JOB=[$2], SAL=[$5]," + " _row_number_transpose_=[ROW_NUMBER() OVER ()])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" - + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM' }, { 'JOB' }, { 'SAL'" + + " LogicalValues(tuples=[[{ 'ENAME' }, { 'COMM ' }, { 'JOB ' }, { 'SAL '" + " }]])\n"; verifyLogical(root, expectedLogical); @@ -261,7 +266,7 @@ public void testTransposeWithLimitColumnName() { verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT TRIM(`column_names`) `column_names`," + "SELECT CAST(TRIM(`column_names`) AS STRING) `column_names`," + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" + " `_row_number_transpose_` = 1) `row 1`," + " MAX(CAST(`_value_transpose_` AS STRING)) FILTER (WHERE" @@ -270,19 +275,19 @@ public void testTransposeWithLimitColumnName() { + " `_row_number_transpose_` = 3) `row 3`\n" + "FROM (SELECT `t`.`ENAME`, `t`.`COMM`, `t`.`JOB`, `t`.`SAL`," + " `t`.`_row_number_transpose_`, `t0`.`column_names`, CASE WHEN `t0`.`column_names` =" - + " 'ENAME' THEN CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column_names` = 'COMM' THEN" - + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column_names` = 'JOB' THEN CAST(`t`.`JOB`" - + " AS STRING) WHEN `t0`.`column_names` = 'SAL' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE" + + " 'ENAME' THEN CAST(`t`.`ENAME` AS STRING) WHEN `t0`.`column_names` = 'COMM ' THEN" + + " NUMBER_TO_STRING(`t`.`COMM`) WHEN `t0`.`column_names` = 'JOB ' THEN CAST(`t`.`JOB`" + + " AS STRING) WHEN `t0`.`column_names` = 'SAL ' THEN NUMBER_TO_STRING(`t`.`SAL`) ELSE" + " NULL END `_value_transpose_`\n" + "FROM (SELECT `ENAME`, `COMM`, `JOB`, `SAL`, ROW_NUMBER() OVER ()" + " `_row_number_transpose_`\n" + "FROM `scott`.`EMP`) `t`\n" + "CROSS JOIN (VALUES ('ENAME'),\n" - + "('COMM'),\n" - + "('JOB'),\n" - + "('SAL')) `t0` (`column_names`)) `t1`\n" + + "('COMM '),\n" + + "('JOB '),\n" + + "('SAL ')) `t0` (`column_names`)) `t1`\n" + "WHERE `t1`.`_value_transpose_` IS NOT NULL\n" - + "GROUP BY TRIM(`column_names`)"; + + "GROUP BY CAST(TRIM(`column_names`) AS STRING)"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/release-notes/opensearch-sql.release-notes-3.7.0.0.md b/release-notes/opensearch-sql.release-notes-3.7.0.0.md index d6898df38e6..7c486d296cf 100644 --- a/release-notes/opensearch-sql.release-notes-3.7.0.0.md +++ b/release-notes/opensearch-sql.release-notes-3.7.0.0.md @@ -32,6 +32,7 @@ Compatible with OpenSearch and OpenSearch Dashboards version 3.7.0 * Create parquet-backed test indices for `spath` command analytics-engine route ([#5441](https://github.com/opensearch-project/sql/pull/5441)) * Improve error messages for invalid index mapping by formatting index patterns and including underlying error details ([#5370](https://github.com/opensearch-project/sql/pull/5370)) * Initial implementation of report-builder interface for richer error context in responses ([#5266](https://github.com/opensearch-project/sql/pull/5266)) +* Validate materialized view subqueries against SQL grammar deny list ([#5485](https://github.com/opensearch-project/sql/pull/5485)) ### Bug Fixes @@ -65,6 +66,7 @@ Compatible with OpenSearch and OpenSearch Dashboards version 3.7.0 * Bump Gradle wrapper to 9.4.1 and workaround `@Ignore` test failure ([#5414](https://github.com/opensearch-project/sql/pull/5414)) * Fix link checker CI failure by excluding LinkedIn URLs ([#5461](https://github.com/opensearch-project/sql/pull/5461)) * Integration test cases for field-level security ([#5008](https://github.com/opensearch-project/sql/pull/5008)) +* Skip `vectorSearch()` missing-plugin integration test when the k-NN plugin is installed, fixing the distribution integ-test run since distributions bundle k-NN ([#5492](https://github.com/opensearch-project/sql/pull/5492)) ### Documentation diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 492f6dee9c6..5b52b9d3387 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -68,7 +68,8 @@ dmlStatement // Primary DML Statements selectStatement - : querySpecification # simpleSelect + : querySpecification # simpleSelect + | querySpecification (UNION ALL querySpecification)+ # unionSelect ; adminStatement diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index ee532a10ed9..aaed2ba5ec2 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -263,6 +263,12 @@ public UnresolvedPlan visitJoinClause(OpenSearchSQLParser.JoinClauseContext ctx) "JOIN is not supported in the V2 SQL engine. Falling back to legacy engine."); } + @Override + public UnresolvedPlan visitUnionSelect(OpenSearchSQLParser.UnionSelectContext ctx) { + throw new SyntaxCheckException( + "UNION is not supported in the V2 SQL engine. Falling back to legacy engine."); + } + @Override public UnresolvedPlan visitHavingClause(HavingClauseContext ctx) { AstHavingFilterBuilder builder = new AstHavingFilterBuilder(context.peek()); diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 273076af40f..33cfb1a56ca 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -222,7 +222,14 @@ public UnresolvedExpression visitWindowFunctionClause(WindowFunctionClauseContex .map(item -> ImmutablePair.of(createSortOption(item), visit(item.expression()))) .collect(Collectors.toList()); } - return new WindowFunction(visit(ctx.function), partitionByList, sortList); + UnresolvedExpression function = visit(ctx.function); + WindowFunction windowFunction = new WindowFunction(function, partitionByList, sortList); + + // Aggregate window with ORDER BY defaults to a running RANGE frame (ranking ignores it). + if (function instanceof AggregateFunction && !sortList.isEmpty()) { + windowFunction.setWindowFrame(WindowFrame.rangeToCurrentRow()); + } + return windowFunction; } @Override diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 7869ba5cdad..9d504a32bb0 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -757,6 +757,13 @@ public UnresolvedPlan visitJoinClause(OpenSearchSQLParser.JoinClauseContext ctx) assertNotNull(new SQLSyntaxParser().parse(query).accept(builder)); } + @Test + public void union_throws_syntax_check_exception() { + assertThrows( + SyntaxCheckException.class, + () -> buildAST("SELECT name FROM t1 UNION ALL SELECT name FROM t2")); + } + @Test public void in_subquery_throws_syntax_check_exception() { assertThrows( diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index e89f2af9b01..cb981f6f45f 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -44,6 +44,8 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ast.expression.WindowFrame; +import org.opensearch.sql.ast.expression.WindowFunction; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxAnalysisErrorListener; @@ -308,12 +310,23 @@ public void canBuildWindowFunctionWithoutOrderBy() { @Test public void canBuildAggregateWindowFunction() { + WindowFunction expected = + new WindowFunction( + aggregate("AVG", qualifiedName("age")), + ImmutableList.of(qualifiedName("state")), + ImmutableList.of(ImmutablePair.of(new SortOption(null, null), qualifiedName("age")))); + expected.setWindowFrame(WindowFrame.rangeToCurrentRow()); + assertEquals(expected, buildExprAst("AVG(age) OVER (PARTITION BY state ORDER BY age)")); + } + + @Test + public void canBuildAggregateWindowFunctionWithoutOrderBy() { assertEquals( window( aggregate("AVG", qualifiedName("age")), ImmutableList.of(qualifiedName("state")), - ImmutableList.of(ImmutablePair.of(new SortOption(null, null), qualifiedName("age")))), - buildExprAst("AVG(age) OVER (PARTITION BY state ORDER BY age)")); + ImmutableList.of()), + buildExprAst("AVG(age) OVER (PARTITION BY state)")); } @Test