From 5156e937a0e55774606b4c546396e20ce3503e2f Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 11 Jun 2025 16:17:23 -0500 Subject: [PATCH 1/5] Add support for updating JacksonEvent array elements Signed-off-by: Taylor Gray --- .../dataprepper/model/event/JacksonEvent.java | 20 ++++++- .../model/event/JacksonEventTest.java | 60 ++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index 741216becc..79505173bd 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -195,8 +195,26 @@ private JsonNode getOrCreateNode(final JsonNode node, final String key) { JsonNode childNode = node.get(key); if (childNode == null) { childNode = mapper.createObjectNode(); - ((ObjectNode) node).set(key, childNode); + if (node.isArray()) { + int index = Integer.parseInt(key); + ArrayNode arrayNode = (ArrayNode) node; + + while (arrayNode.size() <= index) { + arrayNode.addNull(); + } + + JsonNode existing = arrayNode.get(index); + if (existing == null || !existing.isObject()) { + childNode = mapper.createObjectNode(); + arrayNode.set(index, childNode); + } else { + childNode = existing; + } + } else { + ((ObjectNode) node).set(key, childNode); + } } + return childNode; } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index a1fd74b1e1..b68bfe6c4f 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -18,6 +18,7 @@ import java.math.BigDecimal; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -147,6 +148,64 @@ void testPutAndGet_withStrings_eventKey(final String key) { assertThat(result, is(equalTo(value))); } + @Test + void testPutAndGet_withArrays_eventKey() { + + final String key = "list-key/0/foo"; + final String newValue = UUID.randomUUID().toString(); + + final List> listValue = new ArrayList<>(); + final Map mapValue = Map.of("foo", "bar", "foo-2", "bar-2"); + listValue.add(mapValue); + + final String listKey = "list-key"; + final EventKey eventKey = new JacksonEventKey(listKey); + event.put(eventKey, listValue); + + final Map expectedMap = new HashMap<>(); + expectedMap.put(listKey, listValue); + + assertThat(event.toMap(), equalTo(expectedMap)); + + final EventKey eventNestedKey = new JacksonEventKey(key); + event.put(eventNestedKey, newValue); + + final List> newlistValue = new ArrayList<>(); + final Map newMapValue = Map.of("foo", newValue, "foo-2", "bar-2"); + newlistValue.add(newMapValue); + + expectedMap.put(listKey, newlistValue); + + assertThat(event.toMap(), equalTo(expectedMap)); + + + final List> result = event.get(listKey, List.class); + assertThat(result, equalTo(newlistValue)); + + final String resultValue = event.get(key, String.class); + assertThat(resultValue, equalTo(newValue)); + } + + @Test + void testPutAndGet_withArrays_out_of_bounds_creates_new_element() { + + final String key = "list-key/1/foo"; + final String fooValue = UUID.randomUUID().toString(); + + final List> listValue = new ArrayList<>(); + final Map mapValue = Map.of("foo", "bar", "foo-2", "bar-2"); + listValue.add(mapValue); + + final String listKey = "list-key"; + final EventKey eventKey = new JacksonEventKey(listKey); + event.put(eventKey, listValue); + + event.put(key, fooValue); + + final String resultValue = event.get(key, String.class); + assertThat(resultValue, equalTo(fooValue)); + } + @Test public void testPutKeyCannotBeEmptyString() { Throwable exception = assertThrows(IllegalArgumentException.class, () -> event.put("", "value")); @@ -1117,5 +1176,4 @@ private static Stream getBigDecimalPutTestData() { Arguments.of("1.000") ); } - } From 7ef44385b9622e032b715db3e0459d0843c991eb Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 11 Jun 2025 17:20:34 -0500 Subject: [PATCH 2/5] Add integration test for convert type processor Signed-off-by: Taylor Gray --- .../ConvertEntryTypeProcessorTests.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java index bb5cf32983..34d396e876 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorTests.java @@ -389,4 +389,40 @@ void testEmptyKeyConvertEntryTypeProcessor() { when(mockConfig.getKey()).thenReturn(""); assertThrows(IllegalArgumentException.class, () -> new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator)); } + + @Test + void convert_type_on_value_in_array_element_converts_correctly() { + final String eventKey = "list-key/0/foo"; + + when(mockConfig.getType()).thenReturn(TargetType.fromOptionValue("long")); + when(mockConfig.getKey()).thenReturn(eventKey); + + typeConversionProcessor = new ConvertEntryTypeProcessor(pluginMetrics, mockConfig, expressionEvaluator); + + final Map eventData = new HashMap<>(); + + final List> listElement = new ArrayList<>(); + listElement.add(Map.of("foo", 10.0)); + + eventData.put("list-key", listElement); + + final Event event = JacksonEvent.builder() + .withData(eventData) + .withEventType("event") + .build(); + + final List> processedRecords = (List>) typeConversionProcessor.doExecute(Collections.singletonList(new Record<>(event))); + assertThat(processedRecords.size(), equalTo(1)); + + final Event resultEvent = processedRecords.get(0).getData(); + + final Map resultEventData = resultEvent.toMap(); + + final Map expectedEventData = new HashMap<>(); + final List> expectedListElement = new ArrayList<>(); + expectedListElement.add(Map.of("foo", 10L)); + expectedEventData.put("list-key", expectedListElement); + + assertThat(resultEventData, equalTo(expectedEventData)); + } } From 9d23c81e9b072b81e719293fd3d8efc870823374 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 11 Jun 2025 18:35:56 -0500 Subject: [PATCH 3/5] Add additional test case Signed-off-by: Taylor Gray --- .../model/event/JacksonEventTest.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index b68bfe6c4f..5c48811edb 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -187,7 +187,7 @@ void testPutAndGet_withArrays_eventKey() { } @Test - void testPutAndGet_withArrays_out_of_bounds_creates_new_element() { + void testPutAndGet_withArrays_out_of_bounds_on_end_of_list_creates_new_element() { final String key = "list-key/1/foo"; final String fooValue = UUID.randomUUID().toString(); @@ -206,6 +206,34 @@ void testPutAndGet_withArrays_out_of_bounds_creates_new_element() { assertThat(resultValue, equalTo(fooValue)); } + @Test + void testPutAndGet_withArrays_out_of_bounds_creates_new_elements() { + + final String key = "list-key/3/foo"; + final String fooValue = UUID.randomUUID().toString(); + + final List> listValue = new ArrayList<>(); + final Map mapValue = Map.of("foo", "bar", "foo-2", "bar-2"); + listValue.add(mapValue); + + final String listKey = "list-key"; + final EventKey eventKey = new JacksonEventKey(listKey); + event.put(eventKey, listValue); + + event.put(key, fooValue); + + final String resultValue = event.get(key, String.class); + assertThat(resultValue, equalTo(fooValue)); + + final List> listResult = event.get(listKey, List.class); + + assertThat(listResult.size(), equalTo(4)); + assertThat(listResult.get(0), notNullValue()); + assertThat(listResult.get(1), nullValue()); + assertThat(listResult.get(2), nullValue()); + assertThat(listResult.get(3), notNullValue()); + } + @Test public void testPutKeyCannotBeEmptyString() { Throwable exception = assertThrows(IllegalArgumentException.class, () -> event.put("", "value")); From e43373f30bb9ad0b4ffd8dd3c82850367cff6da0 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 11 Jun 2025 19:13:21 -0500 Subject: [PATCH 4/5] Modify to limit number of empty elements to array to only add to the end of the array Signed-off-by: Taylor Gray --- .../dataprepper/model/event/JacksonEvent.java | 8 ++++++++ .../dataprepper/model/event/JacksonEventTest.java | 15 ++------------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index 79505173bd..ab3d2f277c 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -61,6 +61,8 @@ public class JacksonEvent implements Event { private static final Logger LOG = LoggerFactory.getLogger(JacksonEvent.class); + private static final int FILL_OUT_OF_BOUNDS_ELEMENTS_LIMIT = 0; + private static final String SEPARATOR = "/"; private static final ObjectMapper mapper = JsonMapper.builder() @@ -200,6 +202,12 @@ private JsonNode getOrCreateNode(final JsonNode node, final String key) { ArrayNode arrayNode = (ArrayNode) node; while (arrayNode.size() <= index) { + int distanceFromArrayEnd = index - arrayNode.size(); + if (distanceFromArrayEnd >= FILL_OUT_OF_BOUNDS_ELEMENTS_LIMIT + 1) { + throw new IndexOutOfBoundsException( + String.format("Cannot expand array past the limit of size %s to reach index %s", arrayNode.size(), index)); + } + arrayNode.addNull(); } diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index 5c48811edb..49d8385d77 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -207,7 +207,7 @@ void testPutAndGet_withArrays_out_of_bounds_on_end_of_list_creates_new_element() } @Test - void testPutAndGet_withArrays_out_of_bounds_creates_new_elements() { + void testPutAndGet_withArrays_out_of_bounds_throws_IndexOutOfBoundsException() { final String key = "list-key/3/foo"; final String fooValue = UUID.randomUUID().toString(); @@ -220,18 +220,7 @@ void testPutAndGet_withArrays_out_of_bounds_creates_new_elements() { final EventKey eventKey = new JacksonEventKey(listKey); event.put(eventKey, listValue); - event.put(key, fooValue); - - final String resultValue = event.get(key, String.class); - assertThat(resultValue, equalTo(fooValue)); - - final List> listResult = event.get(listKey, List.class); - - assertThat(listResult.size(), equalTo(4)); - assertThat(listResult.get(0), notNullValue()); - assertThat(listResult.get(1), nullValue()); - assertThat(listResult.get(2), nullValue()); - assertThat(listResult.get(3), notNullValue()); + assertThrows(IndexOutOfBoundsException.class, () -> event.put(key, fooValue)); } @Test From 916bd6b2237a9505b5a41fb34ec18820303576e9 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Thu, 12 Jun 2025 11:07:26 -0500 Subject: [PATCH 5/5] Address PR comment Signed-off-by: Taylor Gray --- .../dataprepper/model/event/JacksonEvent.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index ab3d2f277c..5ead432c87 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -201,13 +201,12 @@ private JsonNode getOrCreateNode(final JsonNode node, final String key) { int index = Integer.parseInt(key); ArrayNode arrayNode = (ArrayNode) node; + int distanceFromArrayEnd = index - arrayNode.size(); + if (distanceFromArrayEnd >= FILL_OUT_OF_BOUNDS_ELEMENTS_LIMIT + 1) { + throw new IndexOutOfBoundsException( + String.format("Cannot expand array past the limit of size %s to reach index %s", arrayNode.size(), index)); + } while (arrayNode.size() <= index) { - int distanceFromArrayEnd = index - arrayNode.size(); - if (distanceFromArrayEnd >= FILL_OUT_OF_BOUNDS_ELEMENTS_LIMIT + 1) { - throw new IndexOutOfBoundsException( - String.format("Cannot expand array past the limit of size %s to reach index %s", arrayNode.size(), index)); - } - arrayNode.addNull(); }