From 498c931a79ec48207e98e414e5044f2347c9bb86 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 24 Apr 2025 22:40:58 -0400 Subject: [PATCH 1/4] Introduce RawString type to allow strings that begin with forward slash Signed-off-by: Craig Perkins --- .../src/main/antlr/DataPrepperExpression.g4 | 4 ++++ .../expression/ParseTreeCoercionService.java | 3 +++ .../expression/ParseTreeCoercionServiceTest.java | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 index 8a771a4d4b..6fb7d05d48 100644 --- a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 +++ b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 @@ -269,6 +269,10 @@ VariableNameCharacter | [0-9-] ; +RawString + : [r] DOUBLEQUOTE StringCharacters? DOUBLEQUOTE + ; + String : DOUBLEQUOTE StringCharacters? DOUBLEQUOTE ; diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java index 692a8c561a..87fae9ecf9 100644 --- a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java @@ -70,6 +70,9 @@ public Object coercePrimaryTerminalNode(final TerminalNode node, final Event eve case DataPrepperExpressionParser.String: final String nodeStringValueWithQuotesStripped = nodeStringValue.substring(1, nodeStringValue.length() - 1); return nodeStringValueWithQuotesStripped; + case DataPrepperExpressionParser.RawString: + final String nodeStringValueWithRAndQuotesStripped = nodeStringValue.substring(2, nodeStringValue.length() - 1); + return nodeStringValueWithRAndQuotesStripped; case DataPrepperExpressionParser.Integer: Long longValue = Long.valueOf(nodeStringValue); if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) { diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java index 22477521be..401431e683 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java @@ -226,6 +226,18 @@ void testCoerceTerminalNodeEscapeJsonPointerTypeUnSupportedValues() { assertThrows(ExpressionCoercionException.class, () -> objectUnderTest.coercePrimaryTerminalNode(terminalNode, testEvent)); } + @ParameterizedTest + @MethodSource("provideSupportedJsonPointerValues") + void testCoerceTerminalNodeRawStringTypeSupportedValues(final Object testValue) { + final String testKey = "/testKey"; + final String testRawStringKey = "r\"/testKey\""; + when(token.getType()).thenReturn(DataPrepperExpressionParser.RawString); + when(terminalNode.getSymbol()).thenReturn(token); + when(terminalNode.getText()).thenReturn(testRawStringKey); + final Object result = objectUnderTest.coercePrimaryTerminalNode(terminalNode, null); + assertThat(result, equalTo(testKey)); + } + @Test void testCoerceTerminalNodeJsonPointerTypeUnSupportedValues() { final String testKey1 = "key1"; From 3acd31bae2c6ec4e993449246fb0a564594adba8 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 30 Apr 2025 01:34:40 -0400 Subject: [PATCH 2/4] Add tests in GenericExpressionEvaluator_ConditionalIT Signed-off-by: Craig Perkins --- .../src/main/antlr/DataPrepperExpression.g4 | 1 + .../GenericExpressionEvaluator_ConditionalIT.java | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 index 6fb7d05d48..131f27dbf4 100644 --- a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 +++ b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 @@ -182,6 +182,7 @@ literal | Integer | Boolean | String + | RawString | Null ; diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java index f98546b4dd..09cb587a54 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java @@ -192,6 +192,8 @@ private static Stream validExpressionArguments() { arguments("/should_drop", event("{\"should_drop\": true}"), true), arguments("/should_drop", event("{\"should_drop\": false}"), false), arguments("/logs/2/should_drop", event("{\"logs\": [{}, {}, {\"should_drop\": true}]}"), true), + arguments("/path == " + rawString("/path/to/route"), event("{\"path\": \"/path/to/route\"}"), true), + arguments("/path == " + rawString("/path/to/route"), event("{\"path\": \"/incorrect/path\"}"), false), arguments( escapedJsonPointer(ALL_JACKSON_EVENT_GET_SUPPORTED_CHARACTERS) + " == true", complexEvent(ALL_JACKSON_EVENT_GET_SUPPORTED_CHARACTERS, true), @@ -352,6 +354,10 @@ private static String escapedJsonPointer(final String pointer) { return "\"/" + pointer + "\""; } + private static String rawString(final String original) { + return "r\"" + original + "\""; + } + private static Event event(final String data) { return JacksonEvent.builder().withEventType("event").withData(data).build(); } From 45587ebdbb2b2129120e481a830de4718b16bc2d Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 6 May 2025 13:58:05 -0400 Subject: [PATCH 3/4] Address comments Signed-off-by: Craig Perkins --- .../src/main/antlr/DataPrepperExpression.g4 | 6 +----- .../dataprepper/expression/ParseTreeCoercionService.java | 7 +++---- .../GenericExpressionEvaluator_ConditionalIT.java | 8 ++------ .../expression/ParseTreeCoercionServiceTest.java | 8 ++++---- 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 index 131f27dbf4..05df2b9291 100644 --- a/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 +++ b/data-prepper-expression/src/main/antlr/DataPrepperExpression.g4 @@ -182,7 +182,6 @@ literal | Integer | Boolean | String - | RawString | Null ; @@ -270,12 +269,9 @@ VariableNameCharacter | [0-9-] ; -RawString - : [r] DOUBLEQUOTE StringCharacters? DOUBLEQUOTE - ; - String : DOUBLEQUOTE StringCharacters? DOUBLEQUOTE + | DOUBLEQUOTE DOUBLEQUOTE DOUBLEQUOTE StringCharacters? DOUBLEQUOTE DOUBLEQUOTE DOUBLEQUOTE ; fragment diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java index 87fae9ecf9..7ce1e4fd1c 100644 --- a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java @@ -68,11 +68,10 @@ public Object coercePrimaryTerminalNode(final TerminalNode node, final Event eve case DataPrepperExpressionParser.JsonPointer: return resolveJsonPointerValue(nodeStringValue, event); case DataPrepperExpressionParser.String: - final String nodeStringValueWithQuotesStripped = nodeStringValue.substring(1, nodeStringValue.length() - 1); + final String nodeStringValueWithQuotesStripped = nodeStringValue != null + ? nodeStringValue.replaceAll("^\"{1,3}|\"{1,3}$", "") + : null; return nodeStringValueWithQuotesStripped; - case DataPrepperExpressionParser.RawString: - final String nodeStringValueWithRAndQuotesStripped = nodeStringValue.substring(2, nodeStringValue.length() - 1); - return nodeStringValueWithRAndQuotesStripped; case DataPrepperExpressionParser.Integer: Long longValue = Long.valueOf(nodeStringValue); if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) { diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java index 09cb587a54..ffe2b81020 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenericExpressionEvaluator_ConditionalIT.java @@ -192,8 +192,8 @@ private static Stream validExpressionArguments() { arguments("/should_drop", event("{\"should_drop\": true}"), true), arguments("/should_drop", event("{\"should_drop\": false}"), false), arguments("/logs/2/should_drop", event("{\"logs\": [{}, {}, {\"should_drop\": true}]}"), true), - arguments("/path == " + rawString("/path/to/route"), event("{\"path\": \"/path/to/route\"}"), true), - arguments("/path == " + rawString("/path/to/route"), event("{\"path\": \"/incorrect/path\"}"), false), + arguments("/path == \"\"\"/path/to/route\"\"\"", event("{\"path\": \"/path/to/route\"}"), true), + arguments("/path == \"\"\"/path/to/route\"\"\"", event("{\"path\": \"/incorrect/path\"}"), false), arguments( escapedJsonPointer(ALL_JACKSON_EVENT_GET_SUPPORTED_CHARACTERS) + " == true", complexEvent(ALL_JACKSON_EVENT_GET_SUPPORTED_CHARACTERS, true), @@ -354,10 +354,6 @@ private static String escapedJsonPointer(final String pointer) { return "\"/" + pointer + "\""; } - private static String rawString(final String original) { - return "r\"" + original + "\""; - } - private static Event event(final String data) { return JacksonEvent.builder().withEventType("event").withData(data).build(); } diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java index 401431e683..70e2b33089 100644 --- a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/ParseTreeCoercionServiceTest.java @@ -228,12 +228,12 @@ void testCoerceTerminalNodeEscapeJsonPointerTypeUnSupportedValues() { @ParameterizedTest @MethodSource("provideSupportedJsonPointerValues") - void testCoerceTerminalNodeRawStringTypeSupportedValues(final Object testValue) { + void testCoerceTerminalNodeTripleQuoteStringTypeSupportedValues(final Object testValue) { final String testKey = "/testKey"; - final String testRawStringKey = "r\"/testKey\""; - when(token.getType()).thenReturn(DataPrepperExpressionParser.RawString); + final String testTruipleQuoteStringKey = "\"\"\"/testKey\"\"\""; + when(token.getType()).thenReturn(DataPrepperExpressionParser.String); when(terminalNode.getSymbol()).thenReturn(token); - when(terminalNode.getText()).thenReturn(testRawStringKey); + when(terminalNode.getText()).thenReturn(testTruipleQuoteStringKey); final Object result = objectUnderTest.coercePrimaryTerminalNode(terminalNode, null); assertThat(result, equalTo(testKey)); } From 009d762c092e384ae7190a3663bc5f5cb86b438b Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 6 May 2025 14:23:55 -0400 Subject: [PATCH 4/4] Improve coverage Signed-off-by: Craig Perkins --- .../dataprepper/expression/ParseTreeCoercionService.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java index 7ce1e4fd1c..181b40e9fc 100644 --- a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/ParseTreeCoercionService.java @@ -68,10 +68,7 @@ public Object coercePrimaryTerminalNode(final TerminalNode node, final Event eve case DataPrepperExpressionParser.JsonPointer: return resolveJsonPointerValue(nodeStringValue, event); case DataPrepperExpressionParser.String: - final String nodeStringValueWithQuotesStripped = nodeStringValue != null - ? nodeStringValue.replaceAll("^\"{1,3}|\"{1,3}$", "") - : null; - return nodeStringValueWithQuotesStripped; + return nodeStringValue.replaceAll("^\"{1,3}|\"{1,3}$", ""); case DataPrepperExpressionParser.Integer: Long longValue = Long.valueOf(nodeStringValue); if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) {