Skip to content

Commit d9e47ef

Browse files
ykmr1224asifabashar
ykmr1224
authored and
asifabashar
committed
Support spath with dynamic fields (opensearch-project#5058)
* Support spath with dynamic fields Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add explain test and sql conversion test Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Address comments Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix FieldResolutionResultTest Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Address comment Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix explain Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix join logic to adopt dynamic fields Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix join logic Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix spath.md Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Support fillnull and replace Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Update spath.md Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix test failure Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix doc and error for fillnull Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Accept wildcard only at the end of field list Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Minor fix Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Adopt append command to spath Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix join to allow spath in only one input Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix unit test failure Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix join inputs logic Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix test failure Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Move helper methods Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Add _MAP description in the docs Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Extract more from CalciteRelNodeVisitor Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Refactor IT and address comments Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix FieldResolutionResult Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix DynamicFieldsHelper Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Refactor DynamicFieldsHelper Signed-off-by: Tomoyuki Morita <moritato@amazon.com> * Fix DynamicFieldsHelper Signed-off-by: Tomoyuki Morita <moritato@amazon.com> --------- Signed-off-by: Tomoyuki Morita <moritato@amazon.com>
1 parent 1fb01a9 commit d9e47ef

28 files changed

Lines changed: 1937 additions & 250 deletions

File tree

‎common/src/main/java/org/opensearch/sql/common/utils/DebugUtils.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88
import java.util.Collection;
99
import java.util.Map;
1010
import java.util.stream.Collectors;
11-
import org.apache.logging.log4j.LogManager;
12-
import org.apache.logging.log4j.Logger;
1311

1412
/**
1513
* Utility class for debugging operations. This class is only for debugging purpose, and not
@@ -18,7 +16,6 @@
1816
public class DebugUtils {
1917
// Update this to true while you are debugging. (Safe guard to avoid usage in production code. )
2018
private static final boolean IS_DEBUG = false;
21-
private static final Logger logger = LogManager.getLogger(DebugUtils.class);
2219

2320
public static <T> T debug(T obj, String message) {
2421
verifyDebug();
@@ -39,7 +36,7 @@ private static void verifyDebug() {
3936
}
4037

4138
private static void print(String format, Object... args) {
42-
logger.info(String.format(format, args));
39+
System.out.println(String.format(format, args));
4340
}
4441

4542
private static String getCalledFrom(int pos) {

‎core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionResult.java‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,17 +28,17 @@ public class FieldResolutionResult {
2828
@NonNull private final Set<String> regularFields;
2929
@NonNull private final Wildcard wildcard;
3030

31-
public FieldResolutionResult(Set<String> regularFields) {
31+
public FieldResolutionResult(Collection<String> regularFields) {
3232
this.regularFields = new HashSet<>(regularFields);
3333
this.wildcard = NULL_WILDCARD;
3434
}
3535

36-
public FieldResolutionResult(Set<String> regularFields, Wildcard wildcard) {
36+
public FieldResolutionResult(Collection<String> regularFields, Wildcard wildcard) {
3737
this.regularFields = new HashSet<>(regularFields);
3838
this.wildcard = wildcard;
3939
}
4040

41-
public FieldResolutionResult(Set<String> regularFields, String wildcard) {
41+
public FieldResolutionResult(Collection<String> regularFields, String wildcard) {
4242
this.regularFields = new HashSet<>(regularFields);
4343
this.wildcard = getWildcard(wildcard);
4444
}
@@ -53,12 +53,12 @@ private static Wildcard getWildcard(String wildcard) {
5353
}
5454
}
5555

56-
public FieldResolutionResult(Set<String> regularFields, Set<String> wildcards) {
56+
public FieldResolutionResult(Collection<String> regularFields, Collection<String> wildcards) {
5757
this.regularFields = new HashSet<>(regularFields);
5858
this.wildcard = createOrWildcard(wildcards);
5959
}
6060

61-
private static Wildcard createOrWildcard(Set<String> patterns) {
61+
private static Wildcard createOrWildcard(Collection<String> patterns) {
6262
if (patterns == null || patterns.isEmpty()) {
6363
return NULL_WILDCARD;
6464
}
@@ -70,38 +70,50 @@ private static Wildcard createOrWildcard(Set<String> patterns) {
7070
return new OrWildcard(wildcards);
7171
}
7272

73+
/** Returns unmodifiable view of regular fields. */
7374
public Set<String> getRegularFieldsUnmodifiable() {
7475
return Collections.unmodifiableSet(regularFields);
7576
}
7677

78+
/** Checks if result contains any wildcard patterns. */
7779
public boolean hasWildcards() {
7880
return wildcard != NULL_WILDCARD;
7981
}
8082

83+
/** Checks if result contains partial wildcard patterns (not '*'). */
84+
public boolean hasPartialWildcards() {
85+
return wildcard != NULL_WILDCARD && wildcard != ANY_WILDCARD;
86+
}
87+
88+
/** Checks if result contains regular fields. */
8189
public boolean hasRegularFields() {
8290
return !regularFields.isEmpty();
8391
}
8492

93+
/** Creates new result excluding specified fields. */
8594
public FieldResolutionResult exclude(Collection<String> fields) {
8695
Set<String> combinedFields = new HashSet<>(this.regularFields);
8796
combinedFields.removeAll(fields);
8897
return new FieldResolutionResult(combinedFields, this.wildcard);
8998
}
9099

91-
public FieldResolutionResult or(Set<String> fields) {
100+
/** Creates new result combining this result with additional fields (union). */
101+
public FieldResolutionResult or(Collection<String> fields) {
92102
Set<String> combinedFields = new HashSet<>(this.regularFields);
93103
combinedFields.addAll(fields);
94104
return new FieldResolutionResult(combinedFields, this.wildcard);
95105
}
96106

97-
private Set<String> and(Set<String> fields) {
107+
private Set<String> and(Collection<String> fields) {
98108
return fields.stream()
99109
.filter(field -> this.getRegularFields().contains(field) || this.wildcard.matches(field))
100110
.collect(Collectors.toSet());
101111
}
102112

113+
/** Creates new result intersecting this result with another (intersection). */
103114
public FieldResolutionResult and(FieldResolutionResult other) {
104-
Set<String> combinedFields = this.and(other.regularFields);
115+
Set<String> combinedFields = new HashSet<>();
116+
combinedFields.addAll(this.and(other.regularFields));
105117
combinedFields.addAll(other.and(this.regularFields));
106118

107119
Wildcard combinedWildcard = this.wildcard.and(other.wildcard);
@@ -111,6 +123,7 @@ public FieldResolutionResult and(FieldResolutionResult other) {
111123

112124
/** Interface for wildcard pattern matching. */
113125
public interface Wildcard {
126+
/** Checks if field name matches wildcard pattern. */
114127
boolean matches(String fieldName);
115128

116129
default Wildcard and(Wildcard other) {

‎core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,8 @@
7575
*/
7676
public class FieldResolutionVisitor extends AbstractNodeVisitor<Node, FieldResolutionContext> {
7777

78+
private static final String ALL_FIELDS = "*";
79+
7880
/**
7981
* Analyzes PPL query plan to determine required fields at each node.
8082
*
@@ -110,10 +112,10 @@ private void acceptAndVerifyNodeVisited(Node node, FieldResolutionContext contex
110112

111113
@Override
112114
public Node visitProject(Project node, FieldResolutionContext context) {
113-
boolean isSelectAll =
114-
node.getProjectList().stream().anyMatch(expr -> expr instanceof AllFields);
115+
boolean isSingleSelectAll =
116+
node.getProjectList().size() == 1 && node.getProjectList().get(0) instanceof AllFields;
115117

116-
if (isSelectAll) {
118+
if (isSingleSelectAll) {
117119
visitChildren(node, context);
118120
} else {
119121
Set<String> projectFields = new HashSet<>();
@@ -179,15 +181,14 @@ public Node visitSpath(SPath node, FieldResolutionContext context) {
179181
return visitEval(node.rewriteAsEval(), context);
180182
} else {
181183
// set requirements for spath command;
182-
context.setResult(node, context.getCurrentRequirements());
183184
FieldResolutionResult requirements = context.getCurrentRequirements();
184-
if (requirements.hasWildcards()) {
185+
context.setResult(node, requirements);
186+
if (requirements.hasPartialWildcards()) {
185187
throw new IllegalArgumentException(
186-
"Spath command cannot extract arbitrary fields. Please project fields explicitly by"
187-
+ " fields command without wildcard or stats command.");
188+
"Spath command cannot be used with partial wildcard such as `prefix*`.");
188189
}
189190

190-
context.pushRequirements(context.getCurrentRequirements().or(Set.of(node.getInField())));
191+
context.pushRequirements(requirements.or(Set.of(node.getInField())));
191192
visitChildren(node, context);
192193
context.popRequirements();
193194
return node;
@@ -237,6 +238,8 @@ private Set<String> extractFieldsFromExpression(UnresolvedExpression expr) {
237238

238239
if (expr instanceof Field field) {
239240
fields.add(field.getField().toString());
241+
} else if (expr instanceof AllFields) {
242+
fields.add(ALL_FIELDS);
240243
} else if (expr instanceof QualifiedName name) {
241244
fields.add(name.toString());
242245
} else if (expr instanceof Alias alias) {
@@ -490,43 +493,56 @@ public Node visitStreamWindow(StreamWindow node, FieldResolutionContext context)
490493

491494
@Override
492495
public Node visitFillNull(FillNull node, FieldResolutionContext context) {
496+
if (node.isAgainstAllFields()) {
497+
throw new IllegalArgumentException("Fields need to be specified with fillnull command");
498+
}
499+
Set<String> fields = new HashSet<>();
500+
node.getFields().forEach(field -> fields.addAll(extractFieldsFromExpression(field)));
501+
502+
context.pushRequirements(context.getCurrentRequirements().or(fields));
493503
visitChildren(node, context);
504+
context.popRequirements();
494505
return node;
495506
}
496507

497508
@Override
498509
public Node visitAppendCol(AppendCol node, FieldResolutionContext context) {
499-
visitChildren(node, context);
500-
return node;
510+
throw new IllegalArgumentException(
511+
"AppendCol command cannot be used together with spath command");
501512
}
502513

503514
@Override
504515
public Node visitAppend(Append node, FieldResolutionContext context) {
516+
// dispatch requirements to subsearch and main
517+
acceptAndVerifyNodeVisited(node.getSubSearch(), context);
505518
visitChildren(node, context);
506519
return node;
507520
}
508521

509522
@Override
510523
public Node visitMultisearch(Multisearch node, FieldResolutionContext context) {
511-
visitChildren(node, context);
512-
return node;
524+
throw new IllegalArgumentException(
525+
"Multisearch command cannot be used together with spath command");
513526
}
514527

515528
@Override
516529
public Node visitLookup(Lookup node, FieldResolutionContext context) {
517-
visitChildren(node, context);
518-
return node;
530+
throw new IllegalArgumentException("Lookup command cannot be used together with spath command");
519531
}
520532

521533
@Override
522534
public Node visitValues(Values node, FieldResolutionContext context) {
523-
visitChildren(node, context);
524-
return node;
535+
throw new IllegalArgumentException("Values command cannot be used together with spath command");
525536
}
526537

527538
@Override
528539
public Node visitReplace(Replace node, FieldResolutionContext context) {
540+
Set<String> fields = new HashSet<>();
541+
node.getFieldList().forEach(field -> fields.addAll(extractFieldsFromExpression(field)));
542+
543+
context.pushRequirements(context.getCurrentRequirements().or(fields));
529544
visitChildren(node, context);
545+
context.popRequirements();
530546
return node;
531547
}
532548

@@ -625,6 +641,10 @@ private Set<String> extractFieldsFromAggregation(UnresolvedExpression expr) {
625641
}
626642
}
627643
}
628-
return fields;
644+
return excludeAllFieldsWildcard(fields);
645+
}
646+
647+
private Set<String> excludeAllFieldsWildcard(Set<String> fields) {
648+
return fields.stream().filter(f -> !f.equals(ALL_FIELDS)).collect(Collectors.toSet());
629649
}
630650
}

‎core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@ public List<Field> getFields() {
6363
return getReplacementPairs().stream().map(Pair::getLeft).toList();
6464
}
6565

66+
public boolean isAgainstAllFields() {
67+
return !replacementForAll.isEmpty() && getReplacementPairs().isEmpty();
68+
}
69+
6670
@Override
6771
public FillNull attach(UnresolvedPlan child) {
6872
this.child = child;

‎core/src/main/java/org/opensearch/sql/ast/tree/Join.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.opensearch.sql.ast.AbstractNodeVisitor;
1818
import org.opensearch.sql.ast.expression.Argument;
1919
import org.opensearch.sql.ast.expression.Field;
20+
import org.opensearch.sql.ast.expression.Literal;
2021
import org.opensearch.sql.ast.expression.UnresolvedExpression;
2122

2223
@ToString
@@ -87,6 +88,14 @@ public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
8788
return nodeVisitor.visitJoin(this, context);
8889
}
8990

91+
/**
92+
* @return `overwrite` option value in argumentMap
93+
*/
94+
public boolean isOverwrite() {
95+
return getArgumentMap().get("overwrite") == null // 'overwrite' default value is true
96+
|| getArgumentMap().get("overwrite").equals(Literal.TRUE);
97+
}
98+
9099
public enum JoinType {
91100
INNER,
92101
LEFT,

0 commit comments

Comments
 (0)