diff --git a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 index f3bab6d6b0..30251736c8 100644 --- a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 +++ b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 @@ -167,6 +167,7 @@ fragment FunctionArg : JsonPointer | String + | SUBTRACT? Integer ; variableIdentifier diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java index 4cb6996cd0..5eb2d26e82 100644 --- a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java @@ -134,6 +134,12 @@ private FunctionMetadata parseFunctionMetadata(final String nodeStringValue) { } argList.add(trimmedArg); } else { + try { + argList.add(Integer.parseInt(trimmedArg)); + continue; + } catch (final Exception e) { + + } throw new ExpressionCoercionException(UNSUPPORTED_ARG_TYPE); } } diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java new file mode 100644 index 0000000000..bc569f8c46 --- /dev/null +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/SubListExpressionFunction.java @@ -0,0 +1,68 @@ +package org.opensearch.dataprepper.expression; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import org.opensearch.dataprepper.model.event.Event; + +import javax.inject.Named; + +@Named +public class SubListExpressionFunction implements ExpressionFunction { + + @Override + public String getFunctionName() { + return "subList"; + } + + @Override + public Object evaluate(List args, Event event, Function convertLiteralType) { + if (args.size() != 3) { + throw new IllegalArgumentException("subList() takes 3 arguments"); + } + if (!(args.get(0) instanceof String)) { + throw new IllegalArgumentException("subList() takes 1st argument as string type"); + } + int startIndex, endIndex; + try { + if (args.get(1) instanceof Integer) { + startIndex = (Integer) args.get(1); + } else { + String str = (String) args.get(1); + startIndex = Integer.parseInt(str.substring(1, str.length() - 1)); + } + if (args.get(2) instanceof Integer) { + endIndex = (Integer) args.get(2); + } else { + String str = (String) args.get(2); + endIndex = Integer.parseInt(str.substring(1, str.length() - 1)); + } + } catch (NumberFormatException | ClassCastException e) { + throw new IllegalArgumentException("subList() takes 2nd and 3rd arguments as integers"); + } + + String key = (String)args.get(0); + final Object value = event.get(key, Object.class); + if (value == null) return null; + if (!(value instanceof List)) { + throw new RuntimeException(key + " is not of list type"); + } + List sourceList = (List)value; + + if (endIndex == -1) endIndex = sourceList.size(); + + if (startIndex < 0 || startIndex >= sourceList.size()) { + throw new RuntimeException("subList() start index should be between 0 and list length (inclusive)"); + } + + if (endIndex < 0 || endIndex > sourceList.size()) { + throw new RuntimeException("subList() end index should be between 0 and list length or -1 for list length (exclusive)"); + } + if (startIndex > endIndex) { + throw new RuntimeException("subList() start index should be less than or equal to end index"); + } + return new ArrayList<>(sourceList.subList(startIndex, endIndex)); + } + +} diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java index 0510ca32e2..edc2049063 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java @@ -243,6 +243,7 @@ private static Stream validExpressionArguments() { arguments("/name =~ \".*dataprepper-[0-9]+\"", event("{\"other\": \"dataprepper-abc\"}"), false), arguments("startsWith(\""+strValue+ UUID.randomUUID() + "\",/status)", event("{\"status\":\""+strValue+"\"}"), true), arguments("startsWith(\""+ UUID.randomUUID() +strValue+ "\",/status)", event("{\"status\":\""+strValue+"\"}"), false), + arguments("subList(/list, 1, 2) != null", event("{\"list\": [0, 1, 2, 3]}"), true), arguments("getEventType() == \"event\"", longEvent, true), arguments("getEventType() == \"LOG\"", longEvent, false), arguments("formatDateTime(/time, \"'year='yyyy'/month='MM'/day='dd\", \"UTC-8\") == \"year=2025/month=04/day=01\"", event("{\"time\": " + LocalDateTime.of(2025, 4, 1, 23, 59).toInstant(ZoneOffset.UTC).toEpochMilli() + "}"), true) @@ -317,16 +318,13 @@ private static Stream invalidExpressionSyntaxArguments() { arguments("trueand/status_code", event("{\"status_code\": 200}")), arguments("trueor/status_code", event("{\"status_code\": 200}")), arguments("length(\""+testString+") == "+testStringLength, event("{\"response\": \""+testString+"\"}")), - arguments("hasTags(10)", tagEvent), arguments("hasTags("+ testTag1+")", tagEvent), arguments("hasTags(\""+ testTag1+")", tagEvent), arguments("hasTags(\""+ testTag1+"\","+testTag2+"\")", tagEvent), arguments("hasTags(,\""+testTag2+"\")", tagEvent), arguments("hasTags(\""+testTag2+"\",)", tagEvent), arguments("contains(\""+testTag2+"\",)", tagEvent), - arguments("contains(1234, /strField)", event("{\"intField\":1234,\"strField\":\"string\"}")), arguments("contains(str, /strField)", event("{\"intField\":1234,\"strField\":\"string\"}")), - arguments("contains(/strField, 1234)", event("{\"intField\":1234,\"strField\":\"string\"}")), arguments("/color in {\"blue, \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), arguments("/color in {\"blue\", yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), arguments("/color in {\", \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), @@ -338,13 +336,11 @@ private static Stream invalidExpressionSyntaxArguments() { arguments("/color in {\"\",blue, \"yellow\", \"green\"}", event("{\"color\": \"yellow\"}")), arguments("/value in {22a2.0, 100}", event("{\"value\": 100}")), arguments("/value in {222, 10a0}", event("{\"value\": 100}")), - arguments("getMetadata(10)", tagEvent), arguments("getMetadata("+ testMetadataKey+ ")", tagEvent), arguments("getMetadata(\""+ testMetadataKey+")", tagEvent), arguments("cidrContains(/sourceIp,123)", event("{\"sourceIp\": \"192.0.2.3\"}")), arguments("getEventType() == \"test_event", tagEvent), arguments("getEventType() == test_event\"", tagEvent) - ); } diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java index 67f7f87732..ef1d68e29b 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java @@ -360,7 +360,7 @@ void testCoerceTerminalNodeLengthFunctionWithInvalidArgument() { final String value = RandomStringUtils.randomAlphabetic(10); final Event testEvent = createTestEvent(Map.of(key, value)); when(terminalNode.getSymbol()).thenReturn(token); - when(terminalNode.getText()).thenReturn("length(10)"); + when(terminalNode.getText()).thenReturn("length(arg)"); when(expressionFunctionProvider.provideFunction(eq("length"), any(List.class), any(Event.class), any(Function.class))).thenReturn(value.length()); when(token.getType()).thenReturn(DataPrepperExpressionParser.Function); assertThrows(RuntimeException.class, () -> objectUnderTest.coercePrimaryTerminalNode(terminalNode, testEvent)); diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java new file mode 100644 index 0000000000..d84ec597ce --- /dev/null +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/SubListExpressionFunctionTest.java @@ -0,0 +1,174 @@ +package org.opensearch.dataprepper.expression; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.JacksonEvent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class SubListExpressionFunctionTest { + private SubListExpressionFunction subListExpressionFunction; + private Event testEvent; + private Function testFunction; + + private Event createTestEvent(final Object data) { + return JacksonEvent.builder().withEventType("event").withData(data).build(); + } + + + public SubListExpressionFunction createObjectUnderTest() { + testFunction = mock(Function.class); + return new SubListExpressionFunction(); + } + + @Test + void testFunctionName() { + subListExpressionFunction = createObjectUnderTest(); + assertEquals("subList", subListExpressionFunction.getFunctionName()); + } + + @Test + void testWithValidArguments() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"3\""), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithValidArgumentsCase2() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"3\""), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithValidArgumentsCase3() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("main", Map.of("list", testList))); + assertThat(subListExpressionFunction.evaluate(List.of("/main/list", "\"1\"", "\"3\""), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithValidArgumentsCase4() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("main", Map.of("list", testList))); + assertThat(subListExpressionFunction.evaluate(List.of("/main/list", "\"1\"", "\"-1\""), testEvent, testFunction), equalTo(List.of(2, 3, 4))); + } + + @Test + void testWithValidArgumentsCase5() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/list", 1, 3), testEvent, testFunction), equalTo(List.of(2, 3))); + } + + @Test + void testWithOutOfBoundArgumentsCase1() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"-1\"", "\"4\""), testEvent, testFunction)); + assertEquals("subList() start index should be between 0 and list length (inclusive)", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase2() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"10\"", "\"4\""), testEvent, testFunction)); + assertEquals("subList() start index should be between 0 and list length (inclusive)", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase3() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"4\"", "\"2\""), testEvent, testFunction)); + assertEquals("subList() start index should be less than or equal to end index", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase4() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"10\""), testEvent, testFunction)); + assertEquals("subList() end index should be between 0 and list length or -1 for list length (exclusive)", exception.getMessage()); + } + + @Test + void testWithOutOfBoundArgumentsCase5() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"1\"", "\"-2\""), testEvent, testFunction)); + assertEquals("subList() end index should be between 0 and list length or -1 for list length (exclusive)", exception.getMessage()); + } + + @Test + void testWithInvalidArguments() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"5\"", "\"2\""), testEvent, testFunction)); + assertEquals("subList() start index should be less than or equal to end index", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase2() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"five\"", "\"two\""), testEvent, testFunction)); + assertEquals("subList() takes 2nd and 3rd arguments as integers", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase3() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"0\""), testEvent, testFunction)); + assertEquals("subList() takes 3 arguments", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase4() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of(1, "\"0\"", "\"2\""), testEvent, testFunction)); + assertEquals("subList() takes 1st argument as string type", exception.getMessage()); + } + + @Test + void testWithInvalidArgumentsCase5() { + subListExpressionFunction = createObjectUnderTest(); + testEvent = createTestEvent(Map.of("list", "testList")); + Exception exception = assertThrows(RuntimeException.class, () -> subListExpressionFunction.evaluate(List.of("/list", "\"0\"", "\"2\""), testEvent, testFunction)); + assertEquals("/list is not of list type", exception.getMessage()); + } + + @Test + void testWithUnknownKeyArgument() { + subListExpressionFunction = createObjectUnderTest(); + List testList = List.of(1, 2, 3, 4, 5, 6); + testEvent = createTestEvent(Map.of("list", testList)); + assertThat(subListExpressionFunction.evaluate(List.of("/unknownList", 1, 2), testEvent, testFunction), equalTo(null)); + } +}