Skip to content

Commit 3c95779

Browse files
dlvenabledlvenable
authored
Provides a mechanism to get the size of the JSON representation of an event. (#6635)
Provides a mechanism to get the size of the JSON representation of an event. This adds a new toJsonString() function to expressions. Updates length() function to accept a direct string as input, so that it can be composed with toJsonString(). Includes a fix for add_entries to validate expressions, but not evaluate them in the constructor. The approach was brittle and failed for the new toJsonString function. Resolves #6278. Signed-off-by: David Venable <dlv@amazon.com>
1 parent 2faa175 commit 3c95779

8 files changed

Lines changed: 165 additions & 23 deletions

File tree

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/LengthExpressionFunction.java‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,24 +22,29 @@ public String getFunctionName() {
2222
return "length";
2323
}
2424

25-
public Object evaluate(final List<Object> args, Event event, Function<Object, Object> convertLiteralType) {
25+
public Object evaluate(final List<Object> args, final Event event, final Function<Object, Object> convertLiteralType) {
2626
if (args.size() != 1) {
2727
throw new RuntimeException("length() takes only one argument");
2828
}
29-
Object arg = args.get(0);
30-
if (arg instanceof EventKey) {
31-
EventKey eventKey = (EventKey) arg;
29+
final Object arg = args.get(0);
30+
if (arg instanceof String) {
31+
return getLength((String) arg);
32+
} else if (arg instanceof EventKey) {
33+
final EventKey eventKey = (EventKey) arg;
3234
final Object value = event.get(eventKey, Object.class);
3335
if (value == null) {
3436
return null;
3537
}
3638
if (!(value instanceof String)) {
3739
throw new RuntimeException(eventKey.getKey() + " is not String type");
3840
}
39-
return Integer.valueOf(((String) value).length());
41+
return getLength((String) value);
4042
} else {
4143
throw new RuntimeException("Unexpected argument type: " + arg.getClass());
4244
}
4345
}
44-
}
4546

47+
private static Integer getLength(final String value) {
48+
return value.length();
49+
}
50+
}

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ToJsonStringExpressionFunction.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.expression;
11+
12+
import org.opensearch.dataprepper.model.event.Event;
13+
14+
import javax.inject.Named;
15+
import java.util.List;
16+
import java.util.function.Function;
17+
18+
@Named
19+
public class ToJsonStringExpressionFunction implements ExpressionFunction {
20+
public static final String FUNCTION_NAME = "toJsonString";
21+
22+
public String getFunctionName() {
23+
return FUNCTION_NAME;
24+
}
25+
26+
public Object evaluate(final List<Object> arguments, final Event event, final Function<Object, Object> convertLiteralType) {
27+
if (arguments.size() != 0) {
28+
throw new RuntimeException(FUNCTION_NAME + " takes no arguments");
29+
}
30+
31+
return event.toJsonString();
32+
}
33+
}
34+

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ComposedFunctionsIT.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.time.LocalDateTime;
2020
import java.time.ZoneId;
2121
import java.util.Map;
22+
import java.util.UUID;
2223

2324
import static org.hamcrest.CoreMatchers.equalTo;
2425
import static org.hamcrest.CoreMatchers.instanceOf;
@@ -66,4 +67,20 @@ void evaluate_with_three_levels_using_startsWith_getMetadata_and_formatDateTime(
6667
assertThat(actualValue, instanceOf(Boolean.class));
6768
assertThat(actualValue, equalTo(true));
6869
}
70+
71+
@Test
72+
void evaluate_provides_expected_results_for_composing_length_and_toJsonPointer() {
73+
final GenericExpressionEvaluator objectUnderTest = applicationContext.getBean(GenericExpressionEvaluator.class);
74+
75+
final Map<String, String> dataMap = Map.of(UUID.randomUUID().toString(), UUID.randomUUID().toString());
76+
77+
final Event event = TestEventFactory.getTestEventFactory().eventBuilder(LogEventBuilder.class).withData(dataMap).build();
78+
79+
final String jsonString = event.toJsonString();
80+
81+
final Object actualValue = objectUnderTest.evaluate("length(toJsonString())", event);
82+
83+
assertThat(actualValue, instanceOf(Integer.class));
84+
assertThat((Integer) actualValue, equalTo(jsonString.length()));
85+
}
6986
}

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ private static Stream<Arguments> validExpressionArguments() {
197197
arguments("/durationInNanos > 5000000000", event("{\"durationInNanos\": 6000000000}"), true),
198198
arguments("/response == \"OK\"", event("{\"response\": \"OK\"}"), true),
199199
arguments("length(/response) == "+testStringLength, event("{\"response\": \""+testString+"\"}"), true),
200+
arguments("length(\""+testString+"\") == "+testStringLength, event("{\"response\": \""+testString+"\"}"), true),
200201
arguments("hasTags(\""+ testTag1+"\")", longEvent, true),
201202
arguments("hasTags(\""+ testTag1+"\",\""+testTag2+"\")", longEvent, true),
202203
arguments("hasTags(\""+ testTag1+"\", \""+testTag2+"\", \""+testTag3+"\")", longEvent, true),
@@ -279,7 +280,6 @@ private static Stream<Arguments> invalidExpressionArguments() {
279280
arguments("/status_code != null and /status_code >= 300", event("{\"status_code_not_present\": 200}")),
280281
arguments("(/status_code >= 300) and (/value == 15)", event("{\"status_code2\": 200, \"value\" : 10}")),
281282
arguments("/color in {\"blue\", 222.0, \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")),
282-
arguments("length(\""+testString+"\") == "+testStringLength, event("{\"response\": \""+testString+"\"}")),
283283
arguments("/success < /status_code", event("{\"success\": true, \"status_code\": 200}")),
284284
arguments("/status_code > 3", event("{\"success\": true, \"status_code_not_present\": 200}")),
285285
arguments("/success <= /status_code", event("{\"success\": true, \"status_code\": 200}")),

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/LengthExpressionFunctionTest.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.opensearch.dataprepper.model.event.EventKeyFactory;
2020
import org.opensearch.dataprepper.model.event.JacksonEvent;
2121

22+
import java.util.Collections;
2223
import java.util.List;
2324
import java.util.Map;
2425
import java.util.function.Function;
@@ -47,7 +48,7 @@ public LengthExpressionFunction createObjectUnderTest() {
4748
@ValueSource(ints = {0, 1, 2, 5, 10, 20, 50})
4849
void testWithEventKeyResolvingToString(int stringLength) {
4950
lengthExpressionFunction = createObjectUnderTest();
50-
String testString = RandomStringUtils.randomAlphabetic(stringLength);
51+
final String testString = RandomStringUtils.insecure().nextAlphabetic(stringLength);
5152
testEvent = createTestEvent(Map.of("key", testString));
5253
EventKey eventKey = eventKeyFactory.createEventKey("/key");
5354
assertThat(lengthExpressionFunction.evaluate(List.of(eventKey), testEvent, testFunction), equalTo(testString.length()));
@@ -84,10 +85,13 @@ void testWithUnexpectedArgumentType() {
8485
assertThrows(RuntimeException.class, () -> lengthExpressionFunction.evaluate(List.of(10), testEvent, testFunction));
8586
}
8687

87-
@Test
88-
void testWithStringArgumentThrowsRuntimeException() {
88+
@ParameterizedTest
89+
@ValueSource(ints = {0, 1, 2, 5, 10, 20, 50})
90+
void evaluate_with_a_string_argument(final int stringLength) {
8991
lengthExpressionFunction = createObjectUnderTest();
90-
testEvent = createTestEvent(Map.of("key", "value"));
91-
assertThrows(RuntimeException.class, () -> lengthExpressionFunction.evaluate(List.of("someString"), testEvent, testFunction));
92+
final String testString = RandomStringUtils.insecure().nextAlphabetic(stringLength);
93+
testEvent = createTestEvent(Collections.emptyMap());
94+
assertThat(lengthExpressionFunction.evaluate(List.of(testString), testEvent, testFunction),
95+
equalTo(testString.length()));
9296
}
9397
}

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ToJsonStringExpressionFunctionTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.expression;
11+
12+
import org.junit.jupiter.api.Test;
13+
import org.junit.jupiter.api.extension.ExtendWith;
14+
import org.junit.jupiter.params.ParameterizedTest;
15+
import org.junit.jupiter.params.provider.ValueSource;
16+
import org.mockito.Mock;
17+
import org.mockito.junit.jupiter.MockitoExtension;
18+
import org.opensearch.dataprepper.model.event.Event;
19+
20+
import java.util.Collections;
21+
import java.util.List;
22+
import java.util.UUID;
23+
import java.util.function.Function;
24+
import java.util.stream.Collectors;
25+
import java.util.stream.IntStream;
26+
27+
import static org.hamcrest.CoreMatchers.equalTo;
28+
import static org.hamcrest.MatcherAssert.assertThat;
29+
import static org.junit.jupiter.api.Assertions.assertThrows;
30+
import static org.mockito.Mockito.when;
31+
32+
@ExtendWith(MockitoExtension.class)
33+
class ToJsonStringExpressionFunctionTest {
34+
35+
@Mock
36+
private Event event;
37+
38+
@Mock
39+
private Function<Object, Object> convertLiteralType;
40+
41+
private ToJsonStringExpressionFunction createObjectUnderTest() {
42+
return new ToJsonStringExpressionFunction();
43+
}
44+
45+
@Test
46+
void getFunctionName_returns_functionName() {
47+
assertThat(createObjectUnderTest().getFunctionName(), equalTo("toJsonString"));
48+
}
49+
50+
@ParameterizedTest
51+
@ValueSource(ints = {1, 2, 3, 10})
52+
void evaluate_throws_if_arguments_has_size_greater_than_one(final int numberOfArguments) {
53+
final List<Object> arguments = IntStream.range(0, numberOfArguments)
54+
.mapToObj(i -> "arg" + i)
55+
.collect(Collectors.toList());
56+
57+
final ToJsonStringExpressionFunction objectUnderTest = createObjectUnderTest();
58+
59+
assertThrows(RuntimeException.class, () -> objectUnderTest.evaluate(arguments, event, convertLiteralType));
60+
}
61+
62+
@Test
63+
void evaluate_returns_Event_toJsonString() {
64+
final List<Object> arguments = Collections.emptyList();
65+
66+
final String jsonString = UUID.randomUUID().toString();
67+
when(event.toJsonString()).thenReturn(jsonString);
68+
69+
assertThat(createObjectUnderTest().evaluate(arguments, event, convertLiteralType),
70+
equalTo(jsonString));
71+
}
72+
}

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
48
*/
59

610
package org.opensearch.dataprepper.plugins.processor.mutateevent;
@@ -48,20 +52,16 @@ private static class EntryProperties {
4852
final boolean appendIfExists;
4953
final String addWhen;
5054
final String addToElementWhen;
51-
final Object staticExpressionValue;
52-
5355
EntryProperties(AddEntryProcessorConfig.Entry entry, ExpressionEvaluator evaluator) {
5456
this.overwriteIfExists = entry.getOverwriteIfKeyExists();
5557
this.appendIfExists = entry.getAppendIfKeyExists();
5658
this.addWhen = entry.getAddWhen();
5759
this.addToElementWhen = entry.getAddToElementWhen();
5860
String valueExpr = entry.getValueExpression();
59-
this.staticExpressionValue = (valueExpr != null && !containsEventReference(valueExpr)) ?
60-
evaluator.evaluate(valueExpr, null) : null;
61-
}
62-
63-
private boolean containsEventReference(String expression) {
64-
return expression.contains("/") || expression.contains("getMetadata");
61+
if (valueExpr != null && !evaluator.isValidExpressionStatement(valueExpr)) {
62+
throw new InvalidPluginConfigurationException(
63+
String.format("value_expression \"%s\" is not a valid expression statement.", valueExpr));
64+
}
6565
}
6666
}
6767

@@ -315,9 +315,7 @@ private Object retrieveValue(final AddEntryProcessorConfig.Entry entry, final Ev
315315
KeyInfo keyInfo = preprocessedKeys.get(entryIndex);
316316

317317
if (!Objects.isNull(entry.getValueExpression())) {
318-
value = props.staticExpressionValue != null ?
319-
props.staticExpressionValue :
320-
expressionEvaluator.evaluate(entry.getValueExpression(), context);
318+
value = expressionEvaluator.evaluate(entry.getValueExpression(), context);
321319
} else if (!Objects.isNull(entry.getFormat())) {
322320
try {
323321
if (keyInfo.formatParts != null) {

‎data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
48
*/
59

610
package org.opensearch.dataprepper.plugins.processor.mutateevent;
@@ -253,6 +257,7 @@ public void test_iterate_on_add_value_expression() {
253257
return eventArg.get("testKey", String.class);
254258
});
255259
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry("newMessage", null, 3, null, valueExpression, false, false,null, "message", true, null)));
260+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
256261

257262
final AddEntryProcessor processor = createObjectUnderTest();
258263
final List<Map<String, Object>> mapList = List.of(Map.of("testKey", "testValue"));
@@ -320,6 +325,7 @@ public void testStaticExpressionValueCaching() {
320325
));
321326

322327
when(expressionEvaluator.evaluate(eq(valueExpression), any())).thenReturn(3);
328+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
323329

324330
final AddEntryProcessor processor = createObjectUnderTest();
325331
final Record<Event> record = getEvent("test");
@@ -902,6 +908,7 @@ public void testWithAllValuesNull() {
902908
public void testValueExpressionWithArithmeticExpression() {
903909
String valueExpression = "/number-key";
904910
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry("num_key", null, null, null, valueExpression, false, false,null, null, true, null)));
911+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
905912
final AddEntryProcessor processor = createObjectUnderTest();
906913
final Record<Event> record = getTestEventWithMultipleDataTypes();
907914
Random random = new Random();
@@ -916,6 +923,7 @@ public void testValueExpressionWithArithmeticExpression() {
916923
public void testValueExpressionWithStringExpression() {
917924
String valueExpression = "/string-key";
918925
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry("num_key", null, null, null, valueExpression, false, false,null, null, true, null)));
926+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
919927
final AddEntryProcessor processor = createObjectUnderTest();
920928
final Record<Event> record = getTestEventWithMultipleDataTypes();
921929
String randomString = UUID.randomUUID().toString();
@@ -929,6 +937,7 @@ public void testValueExpressionWithStringExpression() {
929937
public void testValueExpressionWithBooleanExpression() {
930938
String valueExpression = "/number-key > 5";
931939
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry("num_key", null, null, null, valueExpression, false, false,null, null, true, null)));
940+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
932941
final AddEntryProcessor processor = createObjectUnderTest();
933942
final Record<Event> record = getTestEventWithMultipleDataTypes();
934943
when(expressionEvaluator.evaluate(valueExpression, record.getData())).thenReturn(false);
@@ -941,6 +950,7 @@ public void testValueExpressionWithBooleanExpression() {
941950
public void testValueExpressionWithIntegerFunctions() {
942951
String valueExpression = "length(/string-key)";
943952
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry("length_key", null, null, null, valueExpression, false, false,null, null, true, null)));
953+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
944954
final AddEntryProcessor processor = createObjectUnderTest();
945955
final Record<Event> record = getTestEventWithMultipleDataTypes();
946956
String randomString = UUID.randomUUID().toString();
@@ -954,6 +964,7 @@ public void testValueExpressionWithIntegerFunctions() {
954964
public void testValueExpressionWithIntegerFunctionsAndMetadataKey() {
955965
String valueExpression = "length(/date)";
956966
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry(null, "length_key", null, null, valueExpression, false, false,null, null, true, null)));
967+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
957968
final AddEntryProcessor processor = createObjectUnderTest();
958969
final Record<Event> record = getEventWithMetadata("message", Map.of("key", "value"));
959970
String randomString = UUID.randomUUID().toString();
@@ -967,6 +978,7 @@ public void testValueExpressionWithIntegerFunctionsAndMetadataKey() {
967978
public void testValueExpressionWithStringExpressionWithMetadataKey() {
968979
String valueExpression = "/date";
969980
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry(null, "newkey", null, null, valueExpression, false, false,null, null, true, null)));
981+
when(expressionEvaluator.isValidExpressionStatement(valueExpression)).thenReturn(true);
970982
final AddEntryProcessor processor = createObjectUnderTest();
971983
final Record<Event> record = getEventWithMetadata("message", Map.of("key", "value"));
972984
String randomString = UUID.randomUUID().toString();

0 commit comments

Comments
 (0)