Skip to content

Commit 7dac472

Browse files
ishaoxyishaoxy
committed
Signed-off-by: Xinyu Hao <haoxinyu@amazon.com>
2 parents 6e7e8c3 + 00b0f9a commit 7dac472

188 files changed

Lines changed: 1021 additions & 409 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/src/main/java/org/opensearch/sql/ast/tree/Chart.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,16 +97,14 @@ private UnresolvedPlan transformPerFunction() {
9797

9898
PerFunction perFunc = perFuncOpt.get();
9999
// For chart, the rowSplit should contain the span information
100-
UnresolvedExpression spanExpr = rowSplit;
101-
if (rowSplit instanceof Alias) {
102-
spanExpr = ((Alias) rowSplit).getDelegated();
103-
}
100+
UnresolvedExpression spanExpr =
101+
rowSplit instanceof Alias ? ((Alias) rowSplit).getDelegated() : rowSplit;
104102
if (!(spanExpr instanceof Span)) {
105103
return this; // Cannot transform without span information
106104
}
107105

108106
Span span = (Span) spanExpr;
109-
Field spanStartTime = AstDSL.implicitTimestampField();
107+
Field spanStartTime = (Field) span.getField();
110108
Function spanEndTime = timestampadd(span.getUnit(), span.getValue(), spanStartTime);
111109
Function spanMillis = timestampdiff(MILLISECOND, spanStartTime, spanEndTime);
112110
final int SECOND_IN_MILLISECOND = 1000;

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,15 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) {
419419
}
420420
}
421421

422+
if ("LIKE".equalsIgnoreCase(node.getFuncName()) && arguments.size() == 2) {
423+
RexNode defaultCaseSensitive =
424+
CalcitePlanContext.isLegacyPreferred()
425+
? context.rexBuilder.makeLiteral(false)
426+
: context.rexBuilder.makeLiteral(true);
427+
arguments = new ArrayList<>(arguments);
428+
arguments.add(defaultCaseSensitive);
429+
}
430+
422431
RexNode resolvedNode =
423432
PPLFuncImpTable.INSTANCE.resolve(
424433
context.rexBuilder, node.getFuncName(), arguments.toArray(new RexNode[0]));

‎core/src/main/java/org/opensearch/sql/calcite/udf/udaf/LogPatternAggFunction.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public LogParserAccumulator init() {
3939

4040
@Override
4141
public Object result(LogParserAccumulator acc) {
42-
if (acc.size() == 0) {
42+
if (acc.size() == 0 && acc.logSize() == 0) {
4343
return null;
4444
}
4545

@@ -92,7 +92,7 @@ public LogParserAccumulator add(
9292
this.variableCountThreshold = variableCountThreshold;
9393
this.thresholdPercentage = thresholdPercentage;
9494
acc.evaluate(field);
95-
if (bufferLimit > 0 && acc.size() == bufferLimit) {
95+
if (bufferLimit > 0 && acc.logSize() == bufferLimit) {
9696
acc.partialMerge(
9797
maxSampleCount, variableCountThreshold, thresholdPercentage, showNumberedToken);
9898
acc.clearBuffer();
@@ -155,6 +155,10 @@ public static class LogParserAccumulator implements Accumulator {
155155
public Map<String, Map<String, Object>> patternGroupMap = new HashMap<>();
156156

157157
public int size() {
158+
return patternGroupMap.size();
159+
}
160+
161+
public int logSize() {
158162
return logMessages.size();
159163
}
160164

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ public enum BuiltinFunctionName {
193193
GREATER(FunctionName.of(">")),
194194
GTE(FunctionName.of(">=")),
195195
LIKE(FunctionName.of("like")),
196+
ILIKE(FunctionName.of("ilike")),
196197
NOT_LIKE(FunctionName.of("not like")),
197198

198199
/** Aggregation Function. */

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR_OF_DAY;
8080
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF;
8181
import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL;
82+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.ILIKE;
8283
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK;
8384
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM;
8485
import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PARSE;
@@ -332,6 +333,18 @@ default RexNode resolve(RexBuilder builder, RexNode... args) {
332333
}
333334
}
334335

336+
public interface FunctionImp3 extends FunctionImp {
337+
RexNode resolve(RexBuilder builder, RexNode arg1, RexNode arg2, RexNode arg3);
338+
339+
@Override
340+
default RexNode resolve(RexBuilder builder, RexNode... args) {
341+
if (args.length != 3) {
342+
throw new IllegalArgumentException("This function requires exactly 3 arguments");
343+
}
344+
return resolve(builder, args[0], args[1], args[2]);
345+
}
346+
}
347+
335348
/** The singleton instance. */
336349
public static final PPLFuncImpTable INSTANCE;
337350

@@ -1225,17 +1238,22 @@ void populate() {
12251238
arg))),
12261239
PPLTypeChecker.family(SqlTypeFamily.ANY));
12271240
register(
1228-
LIKE,
1241+
ILIKE,
12291242
(FunctionImp2)
12301243
(builder, arg1, arg2) ->
12311244
builder.makeCall(
1232-
SqlLibraryOperators.ILIKE,
1233-
arg1,
1234-
arg2,
1235-
// TODO: Figure out escaping solution. '\\' is used for JSON input but is not
1236-
// necessary for SQL function input
1237-
builder.makeLiteral("\\")),
1245+
SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")),
12381246
PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING));
1247+
register(
1248+
LIKE,
1249+
(FunctionImp3)
1250+
(builder, arg1, arg2, arg3) ->
1251+
((RexLiteral) arg3).getValueAs(Boolean.class)
1252+
? builder.makeCall(
1253+
SqlStdOperatorTable.LIKE, arg1, arg2, builder.makeLiteral("\\"))
1254+
: builder.makeCall(
1255+
SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")),
1256+
PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.BOOLEAN));
12391257
}
12401258
}
12411259

‎core/src/main/java/org/opensearch/sql/expression/function/PatternParserFunctionImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public static Object evalAgg(
107107
@Parameter(name = "field") String field,
108108
@Parameter(name = "aggObject") Object aggObject,
109109
@Parameter(name = "showNumberedToken") Boolean showNumberedToken) {
110-
if (Strings.isBlank(field)) {
110+
if (Strings.isBlank(field) || aggObject == null) {
111111
return EMPTY_RESULT;
112112
}
113113
List<Map<String, Object>> aggResult = (List<Map<String, Object>>) aggObject;

‎core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,8 @@ private static DefaultFunctionResolver gte() {
387387
private static DefaultFunctionResolver like() {
388388
return define(
389389
BuiltinFunctionName.LIKE.getName(),
390-
impl(nullMissingHandling(OperatorUtils::matches), BOOLEAN, STRING, STRING));
390+
impl(nullMissingHandling(OperatorUtils::matches2), BOOLEAN, STRING, STRING),
391+
impl(nullMissingHandling(OperatorUtils::matches3), BOOLEAN, STRING, STRING, BOOLEAN));
391392
}
392393

393394
private static DefaultFunctionResolver regexp() {
@@ -401,10 +402,17 @@ private static DefaultFunctionResolver notLike() {
401402
BuiltinFunctionName.NOT_LIKE.getName(),
402403
impl(
403404
nullMissingHandling(
404-
(v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches(v1, v2))),
405+
(v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches2(v1, v2))),
405406
BOOLEAN,
406407
STRING,
407-
STRING));
408+
STRING),
409+
impl(
410+
nullMissingHandling(
411+
(v1, v2, v3) -> UnaryPredicateOperators.not(OperatorUtils.matches3(v1, v2, v3))),
412+
BOOLEAN,
413+
STRING,
414+
STRING,
415+
BOOLEAN));
408416
}
409417

410418
private static ExprValue lookupTableFunction(

‎core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,31 @@ public class OperatorUtils {
2121
* @param pattern string pattern to match.
2222
* @return if text matches pattern returns true; else return false.
2323
*/
24-
public static ExprBooleanValue matches(ExprValue text, ExprValue pattern) {
24+
public static ExprBooleanValue matches2(ExprValue text, ExprValue pattern) {
2525
return ExprBooleanValue.of(
2626
Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE)
2727
.matcher(text.stringValue())
2828
.matches());
2929
}
3030

31+
/**
32+
* Wildcard pattern matcher util.<br>
33+
* Percent (%) character for wildcard,<br>
34+
* Underscore (_) character for a single character match.
35+
*
36+
* @param pattern string pattern to match.
37+
* @param caseSensitive indicate the case sensitivity of the pattern.
38+
* @return if text matches pattern returns true; else return false.
39+
*/
40+
public static ExprBooleanValue matches3(
41+
ExprValue text, ExprValue pattern, ExprValue caseSensitive) {
42+
Pattern p =
43+
caseSensitive.booleanValue()
44+
? Pattern.compile(patternToRegex(pattern.stringValue()))
45+
: Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE);
46+
return ExprBooleanValue.of(p.matcher(text.stringValue()).matches());
47+
}
48+
3149
/**
3250
* Checks if text matches regular expression pattern.
3351
*

‎core/src/test/java/org/opensearch/sql/data/model/ExprValueCompareTest.java‎

Lines changed: 18 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,26 @@
1212
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING;
1313
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL;
1414
import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime;
15+
import static org.opensearch.sql.utils.DateTimeUtils.extractTimestamp;
1516

16-
import java.time.LocalDate;
17+
import java.time.Instant;
1718
import java.time.Period;
19+
import java.time.ZoneOffset;
1820
import java.util.stream.Stream;
1921
import org.junit.jupiter.api.Test;
2022
import org.junit.jupiter.params.ParameterizedTest;
2123
import org.junit.jupiter.params.provider.Arguments;
2224
import org.junit.jupiter.params.provider.MethodSource;
2325
import org.opensearch.sql.exception.ExpressionEvaluationException;
2426
import org.opensearch.sql.expression.ExpressionTestBase;
27+
import org.opensearch.sql.expression.function.FunctionProperties;
2528

2629
public class ExprValueCompareTest extends ExpressionTestBase {
2730

31+
// Use a fixed timestamp for consistent test results across timezones
32+
private static final FunctionProperties FIXED_FUNCTION_PROPERTIES =
33+
new FunctionProperties(Instant.parse("2023-05-15T00:00:00Z"), ZoneOffset.UTC);
34+
2835
@Test
2936
public void timeValueCompare() {
3037
assertEquals(0, new ExprTimeValue("18:00:00").compareTo(new ExprTimeValue("18:00:00")));
@@ -79,22 +86,18 @@ private static Stream<Arguments> getEqualDatetimeValuesOfDifferentTypes() {
7986
Arguments.of(
8087
new ExprTimestampValue("1984-11-22 00:00:00"), new ExprDateValue("1984-11-22")),
8188
Arguments.of(
82-
new ExprTimestampValue(LocalDate.now() + " 00:00:00"),
83-
new ExprDateValue(LocalDate.now())),
84-
Arguments.of(
85-
new ExprDatetimeValue(LocalDate.now() + " 17:42:15"), new ExprTimeValue("17:42:15")),
89+
new ExprTimestampValue("2023-05-15 00:00:00"), new ExprDateValue("2023-05-15")),
90+
Arguments.of(new ExprDateValue("2023-05-15"), new ExprTimeValue("00:00:00")),
91+
Arguments.of(
92+
new ExprTimestampValue("1984-11-22 00:00:00"), new ExprDateValue("1984-11-22")),
93+
Arguments.of(new ExprTimeValue("17:42:15"), new ExprTimestampValue("2023-05-15 17:42:15")),
8694
Arguments.of(
8795
new ExprDatetimeValue("2012-08-07 19:14:38"),
8896
new ExprTimestampValue("2012-08-07 19:14:38")),
8997
Arguments.of(new ExprDateValue("2012-08-07"), new ExprDatetimeValue("2012-08-07 00:00:00")),
9098
Arguments.of(new ExprDateValue("2007-01-27"), new ExprDatetimeValue("2007-01-27 00:00:00")),
91-
Arguments.of(new ExprDateValue(LocalDate.now()), new ExprTimeValue("00:00:00")),
92-
Arguments.of(
93-
new ExprTimestampValue("1984-11-22 00:00:00"), new ExprDateValue("1984-11-22")),
9499
Arguments.of(
95-
new ExprTimeValue("19:14:38"), new ExprDatetimeValue(LocalDate.now() + " 19:14:38")),
96-
Arguments.of(
97-
new ExprTimeValue("17:42:15"), new ExprTimestampValue(LocalDate.now() + " 17:42:15")));
100+
new ExprTimestampValue("1984-11-22 00:00:00"), new ExprDateValue("1984-11-22")));
98101
}
99102

100103
/**
@@ -106,12 +109,8 @@ private static Stream<Arguments> getEqualDatetimeValuesOfDifferentTypes() {
106109
public void compareEqDifferentDateTimeValueTypes(ExprValue left, ExprValue right) {
107110
assertEquals(
108111
0,
109-
extractDateTime(left, functionProperties)
110-
.compareTo(extractDateTime(right, functionProperties)));
111-
assertEquals(
112-
0,
113-
extractDateTime(right, functionProperties)
114-
.compareTo(extractDateTime(left, functionProperties)));
112+
extractTimestamp(left, FIXED_FUNCTION_PROPERTIES)
113+
.compareTo(extractTimestamp(right, FIXED_FUNCTION_PROPERTIES)));
115114
}
116115

117116
private static Stream<Arguments> getNotEqualDatetimeValuesOfDifferentTypes() {
@@ -120,23 +119,20 @@ private static Stream<Arguments> getNotEqualDatetimeValuesOfDifferentTypes() {
120119
new ExprDatetimeValue("2012-08-07 19:14:38"),
121120
new ExprTimestampValue("1961-04-12 09:07:00")),
122121
Arguments.of(new ExprDatetimeValue("2012-08-07 19:14:38"), new ExprTimeValue("09:07:00")),
123-
Arguments.of(
124-
new ExprDatetimeValue(LocalDate.now() + " 19:14:38"), new ExprTimeValue("09:07:00")),
125122
Arguments.of(new ExprDatetimeValue("2012-08-07 00:00:00"), new ExprDateValue("1961-04-12")),
126123
Arguments.of(new ExprDatetimeValue("1961-04-12 19:14:38"), new ExprDateValue("1961-04-12")),
127124
Arguments.of(new ExprDateValue("1984-11-22"), new ExprDatetimeValue("1961-04-12 19:14:38")),
128125
Arguments.of(
129126
new ExprDateValue("1984-11-22"), new ExprTimestampValue("2020-09-16 17:30:00")),
130127
Arguments.of(new ExprDateValue("1984-11-22"), new ExprTimeValue("19:14:38")),
131-
Arguments.of(new ExprTimeValue("19:14:38"), new ExprDateValue(LocalDate.now())),
132128
Arguments.of(new ExprTimeValue("19:14:38"), new ExprDatetimeValue("2012-08-07 09:07:00")),
129+
Arguments.of(new ExprTimeValue("19:14:38"), new ExprDateValue("2023-05-15")),
133130
Arguments.of(new ExprTimeValue("19:14:38"), new ExprTimestampValue("1984-02-03 04:05:07")),
134131
Arguments.of(
135132
new ExprTimestampValue("2012-08-07 19:14:38"),
136133
new ExprDatetimeValue("1961-04-12 09:07:00")),
137134
Arguments.of(new ExprTimestampValue("2012-08-07 19:14:38"), new ExprTimeValue("09:07:00")),
138-
Arguments.of(
139-
new ExprTimestampValue(LocalDate.now() + " 19:14:38"), new ExprTimeValue("09:07:00")),
135+
Arguments.of(new ExprTimestampValue("2023-05-15 19:14:38"), new ExprTimeValue("09:07:00")),
140136
Arguments.of(
141137
new ExprTimestampValue("2012-08-07 00:00:00"), new ExprDateValue("1961-04-12")),
142138
Arguments.of(

‎core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626

2727
import com.google.common.collect.ImmutableList;
2828
import com.google.common.collect.ImmutableMap;
29+
import java.time.Instant;
30+
import java.time.ZoneOffset;
2931
import java.util.List;
3032
import java.util.function.Function;
3133
import org.opensearch.sql.data.model.ExprValue;
@@ -36,7 +38,9 @@
3638

3739
public class ExpressionTestBase {
3840

39-
protected FunctionProperties functionProperties = new FunctionProperties();
41+
// Use fixed timestamp to ensure timezone-independent tests
42+
protected final FunctionProperties functionProperties =
43+
new FunctionProperties(Instant.parse("2023-05-15T00:00:00Z"), ZoneOffset.UTC);
4044

4145
protected Environment<Expression, ExprType> typeEnv;
4246

0 commit comments

Comments
 (0)