Skip to content

Commit a63e5d8

Browse files
kkondakakkondaka
authored
Support functions in Data Prepper expressions opensearch-project#2626 (opensearch-project#2644)
* Support functions in Data Prepper expressions opensearch-project#2626 Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Addressed review comments. Made ExpressionFunction a interface with provider and implementation Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Added newly created files Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Addressed review comments Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Fixed zero string size issue in LengthExpressionFunction Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Modified to pass Event to ExpressionFunction Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Modified to do argument resolution inside the functions instead of the common infra Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Addressed review comments Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Removed support for literal strings in length() function in dataprepper expressions Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> * Updated the document Signed-off-by: Krishna Kondaka <krishkdk@amazon.com> --------- Signed-off-by: Krishna Kondaka <krishkdk@amazon.com>
1 parent 71415bb commit a63e5d8

15 files changed

Lines changed: 549 additions & 12 deletions

‎data-prepper-expression/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ dependencies {
2828
implementation 'org.apache.logging.log4j:log4j-core'
2929
implementation 'org.apache.logging.log4j:log4j-slf4j2-impl'
3030
testImplementation testLibs.spring.test
31+
testImplementation "org.apache.commons:commons-lang3:3.12.0"
3132
testImplementation 'com.fasterxml.jackson.core:jackson-databind'
3233
}
3334

‎data-prepper-expression/src/main/antlr/DataPrepperExpression.g4‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,14 @@ setInitializer
8686
: LBRACE primary (SET_DELIMITER primary)* RBRACE
8787
;
8888

89-
9089
unaryOperator
9190
: NOT
9291
| SUBTRACT
9392
;
9493

9594
primary
9695
: jsonPointer
96+
| function
9797
| variableIdentifier
9898
| setInitializer
9999
| literal
@@ -104,6 +104,25 @@ jsonPointer
104104
| EscapedJsonPointer
105105
;
106106

107+
function
108+
: Function
109+
;
110+
111+
Function
112+
: JsonPointerCharacters LPAREN FunctionArgs RPAREN
113+
;
114+
115+
fragment
116+
FunctionArgs
117+
: (FunctionArg SPACE* COMMA)* SPACE* FunctionArg
118+
;
119+
120+
fragment
121+
FunctionArg
122+
: JsonPointer
123+
| String
124+
;
125+
107126
variableIdentifier
108127
: variableName
109128
;
@@ -227,7 +246,10 @@ EscapeSequence
227246
: '\\' [btnfr"'\\$]
228247
;
229248

230-
SET_DELIMITER : ',';
249+
SET_DELIMITER
250+
: COMMA
251+
;
252+
COMMA : ',';
231253
EQUAL : '==';
232254
NOT_EQUAL : '!=';
233255
LT : '<';

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ExpressionFunction.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.dataprepper.expression;
7+
8+
import org.opensearch.dataprepper.model.event.Event;
9+
import java.util.List;
10+
import java.util.function.Function;
11+
12+
interface ExpressionFunction {
13+
/**
14+
* @return function name
15+
* @since 2.3
16+
*/
17+
String getFunctionName();
18+
19+
/**
20+
* evaluates the function and returns the result
21+
* @param args list of arguments to the function
22+
* @return the result of function evaluation
23+
* @since 2.3
24+
*/
25+
Object evaluate(final List<Object> args, Event event, Function<Object, Object> convertLiteralType);
26+
}

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ExpressionFunctionProvider.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.dataprepper.expression;
7+
8+
import org.opensearch.dataprepper.model.event.Event;
9+
import javax.inject.Named;
10+
import javax.inject.Inject;
11+
import java.util.Map;
12+
import java.util.List;
13+
import java.util.stream.Collectors;
14+
import java.util.function.Function;
15+
16+
@Named
17+
public class ExpressionFunctionProvider {
18+
private Map<String, ExpressionFunction> expressionFunctionsMap;
19+
20+
@Inject
21+
public ExpressionFunctionProvider(final List<ExpressionFunction> expressionFunctions) {
22+
expressionFunctionsMap = expressionFunctions.stream().collect(Collectors.toMap(e -> e.getFunctionName(), e -> e));
23+
}
24+
25+
public Object provideFunction(final String functionName, final List<Object> argList, Event event, Function<Object, Object> convertLiteralType) {
26+
if (!expressionFunctionsMap.containsKey(functionName)) {
27+
throw new RuntimeException("Unknown function in the expression");
28+
}
29+
return expressionFunctionsMap.get(functionName).evaluate(argList, event, convertLiteralType);
30+
}
31+
32+
}

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/LengthExpressionFunction.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.dataprepper.expression;
7+
8+
import org.opensearch.dataprepper.model.event.Event;
9+
import java.util.List;
10+
import javax.inject.Named;
11+
import java.util.function.Function;
12+
13+
@Named
14+
public class LengthExpressionFunction implements ExpressionFunction {
15+
public String getFunctionName() {
16+
return "length";
17+
}
18+
19+
public Object evaluate(final List<Object> args, Event event, Function<Object, Object> convertLiteralType) {
20+
if (args.size() != 1) {
21+
throw new RuntimeException("length() takes only one argument");
22+
}
23+
Object arg = args.get(0);
24+
if (!(arg instanceof String)) {
25+
throw new RuntimeException("length() takes only String type arguments");
26+
}
27+
String argStr = ((String)arg).trim();
28+
if (argStr.length() == 0) {
29+
return 0;
30+
}
31+
if (argStr.charAt(0) == '\"') {
32+
throw new RuntimeException("Literal strings not supported as arguments to length()");
33+
} else {
34+
// argStr must be JsonPointer
35+
final Object value = event.get(argStr, Object.class);
36+
if (value == null) {
37+
return null;
38+
}
39+
40+
if (!(value instanceof String)) {
41+
throw new RuntimeException(argStr + " is not String type");
42+
}
43+
return Integer.valueOf(((String)value).length());
44+
}
45+
}
46+
}
47+

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,21 +13,54 @@
1313
import javax.inject.Named;
1414
import java.io.Serializable;
1515
import java.util.Map;
16+
import java.util.List;
17+
import java.util.ArrayList;
1618
import java.util.function.Function;
1719

1820
@Named
1921
class ParseTreeCoercionService {
2022
private final Map<Class<? extends Serializable>, Function<Object, Object>> literalTypeConversions;
23+
private ExpressionFunctionProvider expressionFunctionProvider;
24+
private Function<Object, Object> convertLiteralType;
2125

2226
@Inject
23-
public ParseTreeCoercionService(final Map<Class<? extends Serializable>, Function<Object, Object>> literalTypeConversions) {
27+
public ParseTreeCoercionService(final Map<Class<? extends Serializable>, Function<Object, Object>> literalTypeConversions, ExpressionFunctionProvider expressionFunctionProvider) {
2428
this.literalTypeConversions = literalTypeConversions;
29+
convertLiteralType = (value) -> {
30+
if (literalTypeConversions.containsKey(value.getClass())) {
31+
return literalTypeConversions.get(value.getClass()).apply(value);
32+
} else {
33+
throw new ExpressionCoercionException("Unsupported type for value " + value);
34+
}
35+
};
36+
this.expressionFunctionProvider = expressionFunctionProvider;
2537
}
2638

2739
public Object coercePrimaryTerminalNode(final TerminalNode node, final Event event) {
2840
final int nodeType = node.getSymbol().getType();
2941
final String nodeStringValue = node.getText();
3042
switch (nodeType) {
43+
case DataPrepperExpressionParser.Function:
44+
final int funcNameIndex = nodeStringValue.indexOf("(");
45+
final String functionName = nodeStringValue.substring(0, funcNameIndex);
46+
final int argsEndIndex = nodeStringValue.indexOf(")", funcNameIndex);
47+
final String argsStr = nodeStringValue.substring(funcNameIndex+1, argsEndIndex);
48+
final String[] args = argsStr.split(",");
49+
List<Object> argList = new ArrayList<>();
50+
for (final String arg: args) {
51+
String trimmedArg = arg.trim();
52+
if (trimmedArg.charAt(0) == '/') {
53+
argList.add(trimmedArg);
54+
} else if (trimmedArg.charAt(0) == '"') {
55+
if (trimmedArg.charAt(trimmedArg.length()-1) != '"') {
56+
throw new RuntimeException("Invalid string argument. Missing double quote at the end");
57+
}
58+
argList.add(trimmedArg);
59+
} else {
60+
throw new RuntimeException("Unsupported type passed as function argument");
61+
}
62+
}
63+
return expressionFunctionProvider.provideFunction(functionName, argList, event, convertLiteralType);
3164
case DataPrepperExpressionParser.EscapedJsonPointer:
3265
final String jsonPointerWithoutQuotes = nodeStringValue.substring(1, nodeStringValue.length() - 1);
3366
return resolveJsonPointerValue(jsonPointerWithoutQuotes, event);
@@ -65,10 +98,7 @@ private Object resolveJsonPointerValue(final String jsonPointer, final Event eve
6598
final Object value = event.get(jsonPointer, Object.class);
6699
if (value == null) {
67100
return null;
68-
} else if (literalTypeConversions.containsKey(value.getClass())) {
69-
return literalTypeConversions.get(value.getClass()).apply(value);
70-
} else {
71-
throw new ExpressionCoercionException("Unsupported type for value " + value);
72-
}
101+
}
102+
return convertLiteralType.apply(value);
73103
}
74104
}

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ConditionalExpressionEvaluatorIT.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.util.concurrent.Executors;
2424
import java.util.concurrent.TimeUnit;
2525
import java.util.stream.Stream;
26+
import java.util.Random;
2627

2728
import static org.awaitility.Awaitility.await;
2829
import static org.hamcrest.CoreMatchers.equalTo;
@@ -33,6 +34,7 @@
3334
import static org.hamcrest.CoreMatchers.sameInstance;
3435
import static org.hamcrest.MatcherAssert.assertThat;
3536
import static org.junit.jupiter.api.Assertions.assertThrows;
37+
import org.apache.commons.lang3.RandomStringUtils;
3638

3739
class ConditionalExpressionEvaluatorIT {
3840
/**
@@ -127,6 +129,9 @@ private static Stream<Arguments> validExpressionArguments() {
127129
.withData(eventMap)
128130
.build();
129131

132+
Random random = new Random();
133+
int testStringLength = random.nextInt(10);
134+
String testString = RandomStringUtils.randomAlphabetic(testStringLength);
130135
return Stream.of(
131136
Arguments.of("true", event("{}"), true),
132137
Arguments.of("/status_code == 200", event("{\"status_code\": 200}"), true),
@@ -160,11 +165,15 @@ private static Stream<Arguments> validExpressionArguments() {
160165
complexEvent(ALL_JACKSON_EVENT_GET_SUPPORTED_CHARACTERS, true),
161166
true),
162167
Arguments.of("/durationInNanos > 5000000000", event("{\"durationInNanos\": 6000000000}"), true),
163-
Arguments.of("/response == \"OK\"", event("{\"response\": \"OK\"}"), true)
168+
Arguments.of("/response == \"OK\"", event("{\"response\": \"OK\"}"), true),
169+
Arguments.of("length(/response) == "+testStringLength, event("{\"response\": \""+testString+"\"}"), true)
164170
);
165171
}
166172

167173
private static Stream<Arguments> invalidExpressionArguments() {
174+
Random random = new Random();
175+
int testStringLength = random.nextInt(10);
176+
String testString = RandomStringUtils.randomAlphabetic(testStringLength);
168177
return Stream.of(
169178
Arguments.of("/missing", event("{}")),
170179
Arguments.of("/success < /status_code", event("{\"success\": true, \"status_code\": 200}")),
@@ -183,7 +192,9 @@ private static Stream<Arguments> invalidExpressionArguments() {
183192
Arguments.of("not null", event("{}")),
184193
Arguments.of("not/status_code", event("{\"status_code\": 200}")),
185194
Arguments.of("trueand/status_code", event("{\"status_code\": 200}")),
186-
Arguments.of("trueor/status_code", event("{\"status_code\": 200}"))
195+
Arguments.of("trueor/status_code", event("{\"status_code\": 200}")),
196+
Arguments.of("length(\""+testString+") == "+testStringLength, event("{\"response\": \""+testString+"\"}")),
197+
Arguments.of("length(\""+testString+"\") == "+testStringLength, event("{\"response\": \""+testString+"\"}"))
187198
);
188199
}
189200

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ExpressionFunctionProviderTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.dataprepper.expression;
7+
8+
import org.opensearch.dataprepper.model.event.Event;
9+
import org.junit.jupiter.api.Test;
10+
import org.junit.jupiter.api.extension.ExtendWith;
11+
import org.mockito.junit.jupiter.MockitoExtension;
12+
import org.apache.commons.lang3.RandomStringUtils;
13+
import static org.hamcrest.CoreMatchers.equalTo;
14+
import static org.hamcrest.MatcherAssert.assertThat;
15+
import static org.junit.jupiter.api.Assertions.assertThrows;
16+
import static org.mockito.ArgumentMatchers.any;
17+
import static org.mockito.Mockito.mock;
18+
import static org.mockito.Mockito.lenient;
19+
20+
import java.util.List;
21+
import java.util.function.Function;
22+
23+
@ExtendWith(MockitoExtension.class)
24+
class ExpressionFunctionProviderTest {
25+
private ExpressionFunctionProvider objectUnderTest;
26+
private String testFunctionName;
27+
private Object testResultObject;
28+
private ExpressionFunction expressionFunction;
29+
private Function<Object, Object> testFunction;
30+
private Event testEvent;
31+
32+
public ExpressionFunctionProvider createObjectUnderTest() {
33+
expressionFunction = mock(ExpressionFunction.class);
34+
testFunction = mock(Function.class);
35+
testFunctionName = RandomStringUtils.randomAlphabetic(8);
36+
testEvent = mock(Event.class);
37+
testResultObject = mock(Object.class);
38+
lenient().when(expressionFunction.evaluate(any(List.class), any(Event.class), any(Function.class))).thenReturn(testResultObject);
39+
lenient().when(expressionFunction.getFunctionName()).thenReturn(testFunctionName);
40+
41+
return new ExpressionFunctionProvider(List.of(expressionFunction));
42+
}
43+
44+
@Test
45+
void testUnknownFunction() {
46+
objectUnderTest = createObjectUnderTest();
47+
String unknownFunctionName = RandomStringUtils.randomAlphabetic(8);
48+
assertThrows(RuntimeException.class, () -> objectUnderTest.provideFunction(unknownFunctionName, List.of(), testEvent, testFunction));
49+
}
50+
51+
@Test
52+
void testFunctionBasic() {
53+
objectUnderTest = createObjectUnderTest();
54+
assertThat(objectUnderTest.provideFunction(testFunctionName, List.of(), testEvent, testFunction), equalTo(testResultObject));
55+
}
56+
57+
}

0 commit comments

Comments
 (0)