From 09d6495ec57b65832253eeff5adf45246b1c4fa3 Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Mon, 23 Jun 2025 15:07:51 +0000 Subject: [PATCH 1/3] Add support for convering keys to lowercase/uppercase in RenameKeyProcessor Signed-off-by: Krishna Kondaka --- .../dataprepper/common}/TransformOption.java | 4 +- .../common}/TransformOptionTest.java | 4 +- .../processor/keyvalue/KeyValueProcessor.java | 1 + .../keyvalue/KeyValueProcessorConfig.java | 1 + .../keyvalue/KeyValueProcessorTests.java | 1 + .../mutateevent/RenameKeyProcessor.java | 30 ++++++++++++ .../mutateevent/RenameKeyProcessorConfig.java | 18 ++++++- .../mutateevent/RenameKeyProcessorTests.java | 47 +++++++++++++++++++ 8 files changed, 100 insertions(+), 6 deletions(-) rename data-prepper-plugins/{key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue => common/src/main/java/org/opensearch/dataprepper/common}/TransformOption.java (92%) rename data-prepper-plugins/{key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue => common/src/test/java/org/opensearch/dataprepper/common}/TransformOptionTest.java (98%) diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java similarity index 92% rename from data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java rename to data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java index 991d087830..96b171530f 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.keyvalue; +package org.opensearch.dataprepper.common; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; @@ -39,7 +39,7 @@ public String getTransformName() { return transformName; } - Function getTransformFunction() { + public Function getTransformFunction() { return transformFunction; } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java similarity index 98% rename from data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java rename to data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java index a39ea8dbc3..eb52ece1ef 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.keyvalue; +package org.opensearch.dataprepper.common; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -88,4 +88,4 @@ public Stream provideArguments(final ExtensionContext exten ); } } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index c7203bbbc1..5abbca21ae 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -10,6 +10,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index 0d1d2154cf..6ae3c853ff 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -13,6 +13,7 @@ import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Size; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 76ada6d76b..727e6ec89f 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -16,6 +16,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java index 6699f409cb..019700a6ce 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java @@ -11,6 +11,7 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.processor.AbstractProcessor; @@ -32,13 +33,23 @@ public class RenameKeyProcessor extends AbstractProcessor, Record< private final List entries; private final ExpressionEvaluator expressionEvaluator; + private final TransformOption transformOption; @DataPrepperPluginConstructor public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProcessorConfig config, final ExpressionEvaluator expressionEvaluator) { super(pluginMetrics); this.entries = config.getEntries(); this.expressionEvaluator = expressionEvaluator; + this.transformOption = config.getTransformOption(); + boolean entriesProvided = config.getEntries() != null; + boolean transformOptionProvided = (transformOption != null && transformOption != TransformOption.NONE); + if ((!entriesProvided ^ transformOptionProvided)) { + throw new InvalidPluginConfigurationException("Only one of 'entries' or 'transform_key' should be specified."); + } + if (config.getEntries() == null || config.getEntries().size() == 0) { + return; + } config.getEntries().forEach(entry -> { if (entry.getRenameWhen() != null && !expressionEvaluator.isValidExpressionStatement(entry.getRenameWhen())) { @@ -55,6 +66,20 @@ public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProc }); } + private void transformEvent(final Event event, Map map, final String keyPrefix) { + for (Map.Entry entry : map.entrySet()) { + Object result = null; + try { + if (entry.getValue() instanceof Map) { + transformEvent(event, (Map)entry.getValue(), keyPrefix+entry.getKey()+"/"); + } + Object value = event.get(keyPrefix+entry.getKey(), Object.class); + event.delete(keyPrefix+entry.getKey()); + event.put(keyPrefix+transformOption.getTransformFunction().apply(entry.getKey()), value); + } catch (Exception ignored) {} + } + } + @Override public Collection> doExecute(final Collection> records) { for(final Record record : records) { @@ -62,6 +87,11 @@ public Collection> doExecute(final Collection> recor try { + if (transformOption != null && transformOption != TransformOption.NONE) { + transformEvent(recordEvent, recordEvent.toMap(), ""); + continue; + } + for (RenameKeyProcessorConfig.Entry entry : entries) { if (Objects.nonNull(entry.getRenameWhen()) && !expressionEvaluator.evaluateConditional(entry.getRenameWhen(), recordEvent)) { continue; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java index 87ff23247e..a4e25f5f90 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java @@ -13,6 +13,7 @@ import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; @@ -116,12 +117,25 @@ public Entry() { } } - @NotEmpty - @NotNull @Valid + @AlsoRequired(values = { + @AlsoRequired.Required(name = "transform_key", allowedValues = {"null", "none"}) + }) private List entries; + @JsonProperty(value = "transform_key", defaultValue = "none") + @JsonPropertyDescription("Allows transforming the key's name such as making the name all lowercase.") + @AlsoRequired(values = { + @AlsoRequired.Required(name = "entries", allowedValues = {"null"}) + }) + private TransformOption transformOption = TransformOption.NONE; + public List getEntries() { return entries; } + + public TransformOption getTransformOption() { + return transformOption; + } + } diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java index d9d0c67dc4..e4dabea557 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java @@ -12,6 +12,7 @@ import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.record.Record; import org.junit.jupiter.api.Test; @@ -224,7 +225,53 @@ public void testNoRename_when_RenameWhen_returns_false() { assertThat(editedRecords.get(0).getData().get("message", Object.class), equalTo("thisisamessage")); } + @Test + public void test_transformKey_converting_allkeys_lowercase() { + Map data = Map.of("KeY1", 1, "kEy2", Map.of("keY3", Map.of("key4", "value4", "KEY5", 5.555))); + when(mockConfig.getEntries()).thenReturn(null); + when(mockConfig.getTransformOption()).thenReturn(TransformOption.LOWERCASE); + Record record = buildRecordWithEvent(data); + final RenameKeyProcessor processor = createObjectUnderTest(); + final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey("key1"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("key2"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("key2/key3"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("key2/key3/key4"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("key2/key3/key5"), is(true)); + } + @Test + public void test_transformKey_converting_allkeys_uppercase() { + Map data = Map.of("KeY1", 1, "kEy2", Map.of("keY3", Map.of("key4", "value4", "KEY5", 5.555))); + when(mockConfig.getEntries()).thenReturn(null); + when(mockConfig.getTransformOption()).thenReturn(TransformOption.UPPERCASE); + Record record = buildRecordWithEvent(data); + final RenameKeyProcessor processor = createObjectUnderTest(); + final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey("KEY1"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("KEY2"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3/KEY4"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3/KEY5"), is(true)); + } + + @Test + public void test_transformKey_converting_allkeys_capitalize() { + Map data = Map.of("key1", 1, "key2", Map.of("key3", Map.of("key4", "value4", "Key5", 5.555))); + when(mockConfig.getEntries()).thenReturn(null); + when(mockConfig.getTransformOption()).thenReturn(TransformOption.CAPITALIZE); + Record record = buildRecordWithEvent(data); + final RenameKeyProcessor processor = createObjectUnderTest(); + final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey("Key1"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key4"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key5"), is(true)); + } private RenameKeyProcessor createObjectUnderTest() { return new RenameKeyProcessor(pluginMetrics, mockConfig, expressionEvaluator); From d5b740e78500aea599ebc8b9fb41353fa5c0a5ff Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Tue, 24 Jun 2025 00:26:36 +0000 Subject: [PATCH 2/3] Addressed review comments Signed-off-by: Krishna Kondaka --- .../mutate-event-processors/build.gradle | 3 +- .../mutateevent/RenameKeyProcessor.java | 31 +++---- .../mutateevent/RenameKeyProcessorConfig.java | 7 ++ .../mutateevent/RenameKeyProcessorTests.java | 91 ++++++++++++++++--- 4 files changed, 99 insertions(+), 33 deletions(-) diff --git a/data-prepper-plugins/mutate-event-processors/build.gradle b/data-prepper-plugins/mutate-event-processors/build.gradle index 802277ac21..fc419ce02b 100644 --- a/data-prepper-plugins/mutate-event-processors/build.gradle +++ b/data-prepper-plugins/mutate-event-processors/build.gradle @@ -25,4 +25,5 @@ dependencies { testImplementation project(':data-prepper-test:test-event') testImplementation testLibs.slf4j.simple testImplementation testLibs.spring.test -} \ No newline at end of file + testImplementation project(':data-prepper-test:test-common') +} diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java index 019700a6ce..597221afd0 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java @@ -42,33 +42,26 @@ public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProc this.expressionEvaluator = expressionEvaluator; this.transformOption = config.getTransformOption(); - boolean entriesProvided = config.getEntries() != null; - boolean transformOptionProvided = (transformOption != null && transformOption != TransformOption.NONE); - if ((!entriesProvided ^ transformOptionProvided)) { - throw new InvalidPluginConfigurationException("Only one of 'entries' or 'transform_key' should be specified."); - } - if (config.getEntries() == null || config.getEntries().size() == 0) { - return; - } - config.getEntries().forEach(entry -> { - if (entry.getRenameWhen() != null + if (config.getEntries() != null) { + config.getEntries().forEach(entry -> { + if (entry.getRenameWhen() != null && !expressionEvaluator.isValidExpressionStatement(entry.getRenameWhen())) { throw new InvalidPluginConfigurationException( String.format("rename_when %s is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", entry.getRenameWhen())); - } - if (entry.getFromKey() == null && entry.getFromKeyPattern() == null) { - throw new InvalidPluginConfigurationException("Either from_key or from_key_regex must be specified. "); - } - if (entry.getFromKey() != null && entry.getFromKeyPattern() != null) { - throw new InvalidPluginConfigurationException("Only one of from_key or from_key_regex should be specified."); - } - }); + } + if (entry.getFromKey() == null && entry.getFromKeyPattern() == null) { + throw new InvalidPluginConfigurationException("Either from_key or from_key_regex must be specified. "); + } + if (entry.getFromKey() != null && entry.getFromKeyPattern() != null) { + throw new InvalidPluginConfigurationException("Only one of from_key or from_key_regex should be specified."); + } + }); + } } private void transformEvent(final Event event, Map map, final String keyPrefix) { for (Map.Entry entry : map.entrySet()) { - Object result = null; try { if (entry.getValue() instanceof Map) { transformEvent(event, (Map)entry.getValue(), keyPrefix+entry.getKey()+"/"); diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java index a4e25f5f90..7d25bb456c 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -138,4 +139,10 @@ public TransformOption getTransformOption() { return transformOption; } + @AssertTrue(message = "entries and transform_key are mutually exclusive options. entries or transform_key is required.") + boolean isValidConfig() { + return (transformOption != null && transformOption != TransformOption.NONE) ^ entries != null; + } + + } diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java index e4dabea557..8049d18f1d 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java @@ -15,6 +15,8 @@ import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.opensearch.dataprepper.common.TransformOption; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -34,6 +36,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -60,6 +64,25 @@ void invalid_rename_when_throws_InvalidPluginConfigurationException() { assertThrows(InvalidPluginConfigurationException.class, this::createObjectUnderTest); } + @Test + void invalid_config_tests_with_entries_and_transform_options() throws NoSuchFieldException, IllegalAccessException { + final String renameWhen = UUID.randomUUID().toString(); + List entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen)); + + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.LOWERCASE); + assertFalse(renameKeyProcessorConfig.isValidConfig()); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); + assertTrue(renameKeyProcessorConfig.isValidConfig()); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.LOWERCASE); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null); + assertTrue(renameKeyProcessorConfig.isValidConfig()); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); + assertFalse(renameKeyProcessorConfig.isValidConfig()); + } + @Test void invalid_config_when_both_from_key_empty_throws_InvalidPluginConfigurationException() { when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry(null,null, "newMessage", true, null))); @@ -227,50 +250,92 @@ public void testNoRename_when_RenameWhen_returns_false() { @Test public void test_transformKey_converting_allkeys_lowercase() { - Map data = Map.of("KeY1", 1, "kEy2", Map.of("keY3", Map.of("key4", "value4", "KEY5", 5.555))); + final String key1 = "KeY1"; + final String key2 = "kEy2"; + final String key3 = "keY3"; + final String key4 = "key4"; + final String key5 = "KEY5"; + Map data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555))); when(mockConfig.getEntries()).thenReturn(null); when(mockConfig.getTransformOption()).thenReturn(TransformOption.LOWERCASE); Record record = buildRecordWithEvent(data); final RenameKeyProcessor processor = createObjectUnderTest(); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); assertThat(editedRecords.size(), equalTo(1)); - assertThat(editedRecords.get(0).getData().containsKey("key1"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("key2"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("key2/key3"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("key2/key3/key4"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("key2/key3/key5"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key1), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false)); + + assertThat(editedRecords.get(0).getData().containsKey(key1.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key1.toLowerCase(), Integer.class), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key4.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key4.toLowerCase(), String.class), equalTo("value4")); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key5.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key5.toLowerCase(), String.class), equalTo("5.555")); } @Test public void test_transformKey_converting_allkeys_uppercase() { - Map data = Map.of("KeY1", 1, "kEy2", Map.of("keY3", Map.of("key4", "value4", "KEY5", 5.555))); + final String key1 = "KeY1"; + final String key2 = "kEy2"; + final String key3 = "keY3"; + final String key4 = "key4"; + final String key5 = "KEY5"; + Map data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555))); when(mockConfig.getEntries()).thenReturn(null); when(mockConfig.getTransformOption()).thenReturn(TransformOption.UPPERCASE); Record record = buildRecordWithEvent(data); final RenameKeyProcessor processor = createObjectUnderTest(); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); assertThat(editedRecords.size(), equalTo(1)); - assertThat(editedRecords.get(0).getData().containsKey("KEY1"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("KEY2"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3/KEY4"), is(true)); - assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3/KEY5"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key1), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false)); + + assertThat(editedRecords.get(0).getData().containsKey(key1.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key1.toUpperCase(), Integer.class), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key4.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key4.toUpperCase(), String.class), equalTo("value4")); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key5.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key5.toUpperCase(), String.class), equalTo("5.555")); } @Test public void test_transformKey_converting_allkeys_capitalize() { - Map data = Map.of("key1", 1, "key2", Map.of("key3", Map.of("key4", "value4", "Key5", 5.555))); + final String key1 = "key1"; + final String key2 = "key2"; + final String key3 = "key3"; + final String key4 = "key4"; + final String key5 = "key5"; + Map data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555))); + when(mockConfig.getEntries()).thenReturn(null); when(mockConfig.getTransformOption()).thenReturn(TransformOption.CAPITALIZE); Record record = buildRecordWithEvent(data); final RenameKeyProcessor processor = createObjectUnderTest(); final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key1), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false)); + assertThat(editedRecords.get(0).getData().containsKey("Key1"), is(true)); assertThat(editedRecords.get(0).getData().containsKey("Key2"), is(true)); assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3"), is(true)); assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key4"), is(true)); + assertThat(editedRecords.get(0).getData().get("Key2/Key3/Key4", String.class), equalTo("value4")); assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key5"), is(true)); + assertThat(editedRecords.get(0).getData().get("Key2/Key3/Key5", String.class), equalTo("5.555")); } private RenameKeyProcessor createObjectUnderTest() { From 578bc4959ca747a798382df33a2efbcee807d39d Mon Sep 17 00:00:00 2001 From: Krishna Kondaka Date: Tue, 24 Jun 2025 16:14:50 +0000 Subject: [PATCH 3/3] Addressed review comments Signed-off-by: Krishna Kondaka --- .../mutateevent/RenameKeyProcessorTests.java | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java index 8049d18f1d..8dba0d23cc 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java @@ -65,7 +65,7 @@ void invalid_rename_when_throws_InvalidPluginConfigurationException() { } @Test - void invalid_config_tests_with_entries_and_transform_options() throws NoSuchFieldException, IllegalAccessException { + void invalid_config_tests_with_entries_and_transform_options_both_present() throws NoSuchFieldException, IllegalAccessException { final String renameWhen = UUID.randomUUID().toString(); List entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen)); @@ -74,13 +74,32 @@ void invalid_config_tests_with_entries_and_transform_options() throws NoSuchFiel ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries); ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.LOWERCASE); assertFalse(renameKeyProcessorConfig.isValidConfig()); + } + + @Test + void invalid_config_tests_with_entries_and_transform_options_both_not_present() throws NoSuchFieldException, IllegalAccessException { + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); + assertFalse(renameKeyProcessorConfig.isValidConfig()); + } + + @Test + void invalid_config_tests_with_entries_and_transform_options_valid_entries() throws NoSuchFieldException, IllegalAccessException { + final String renameWhen = UUID.randomUUID().toString(); + List entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen)); + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries); ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); assertTrue(renameKeyProcessorConfig.isValidConfig()); - ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.LOWERCASE); + } + + @Test + void invalid_config_tests_with_entries_and_transform_options_valid_transformation() throws NoSuchFieldException, IllegalAccessException { + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.UPPERCASE); assertTrue(renameKeyProcessorConfig.isValidConfig()); - ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); - assertFalse(renameKeyProcessorConfig.isValidConfig()); } @Test