diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java similarity index 92% rename from data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java rename to data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java index 991d087830..96b171530f 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.keyvalue; +package org.opensearch.dataprepper.common; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; @@ -39,7 +39,7 @@ public String getTransformName() { return transformName; } - Function getTransformFunction() { + public Function getTransformFunction() { return transformFunction; } diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java similarity index 98% rename from data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java rename to data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java index a39ea8dbc3..eb52ece1ef 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.dataprepper.plugins.processor.keyvalue; +package org.opensearch.dataprepper.common; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtensionContext; @@ -88,4 +88,4 @@ public Stream provideArguments(final ExtensionContext exten ); } } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java index c7203bbbc1..5abbca21ae 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java @@ -10,6 +10,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; diff --git a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java index 0d1d2154cf..6ae3c853ff 100644 --- a/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java +++ b/data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java @@ -13,6 +13,7 @@ import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Size; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; diff --git a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java index 76ada6d76b..727e6ec89f 100644 --- a/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java +++ b/data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java @@ -16,6 +16,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.JacksonEvent; diff --git a/data-prepper-plugins/mutate-event-processors/build.gradle b/data-prepper-plugins/mutate-event-processors/build.gradle index 802277ac21..fc419ce02b 100644 --- a/data-prepper-plugins/mutate-event-processors/build.gradle +++ b/data-prepper-plugins/mutate-event-processors/build.gradle @@ -25,4 +25,5 @@ dependencies { testImplementation project(':data-prepper-test:test-event') testImplementation testLibs.slf4j.simple testImplementation testLibs.spring.test -} \ No newline at end of file + testImplementation project(':data-prepper-test:test-common') +} diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java index 6699f409cb..597221afd0 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java @@ -11,6 +11,7 @@ import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.processor.AbstractProcessor; @@ -32,27 +33,44 @@ public class RenameKeyProcessor extends AbstractProcessor, Record< private final List entries; private final ExpressionEvaluator expressionEvaluator; + private final TransformOption transformOption; @DataPrepperPluginConstructor public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProcessorConfig config, final ExpressionEvaluator expressionEvaluator) { super(pluginMetrics); this.entries = config.getEntries(); this.expressionEvaluator = expressionEvaluator; + this.transformOption = config.getTransformOption(); - config.getEntries().forEach(entry -> { - if (entry.getRenameWhen() != null + if (config.getEntries() != null) { + config.getEntries().forEach(entry -> { + if (entry.getRenameWhen() != null && !expressionEvaluator.isValidExpressionStatement(entry.getRenameWhen())) { throw new InvalidPluginConfigurationException( String.format("rename_when %s is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", entry.getRenameWhen())); - } - if (entry.getFromKey() == null && entry.getFromKeyPattern() == null) { - throw new InvalidPluginConfigurationException("Either from_key or from_key_regex must be specified. "); - } - if (entry.getFromKey() != null && entry.getFromKeyPattern() != null) { - throw new InvalidPluginConfigurationException("Only one of from_key or from_key_regex should be specified."); - } - }); + } + if (entry.getFromKey() == null && entry.getFromKeyPattern() == null) { + throw new InvalidPluginConfigurationException("Either from_key or from_key_regex must be specified. "); + } + if (entry.getFromKey() != null && entry.getFromKeyPattern() != null) { + throw new InvalidPluginConfigurationException("Only one of from_key or from_key_regex should be specified."); + } + }); + } + } + + private void transformEvent(final Event event, Map map, final String keyPrefix) { + for (Map.Entry entry : map.entrySet()) { + try { + if (entry.getValue() instanceof Map) { + transformEvent(event, (Map)entry.getValue(), keyPrefix+entry.getKey()+"/"); + } + Object value = event.get(keyPrefix+entry.getKey(), Object.class); + event.delete(keyPrefix+entry.getKey()); + event.put(keyPrefix+transformOption.getTransformFunction().apply(entry.getKey()), value); + } catch (Exception ignored) {} + } } @Override @@ -62,6 +80,11 @@ public Collection> doExecute(final Collection> recor try { + if (transformOption != null && transformOption != TransformOption.NONE) { + transformEvent(recordEvent, recordEvent.toMap(), ""); + continue; + } + for (RenameKeyProcessorConfig.Entry entry : entries) { if (Objects.nonNull(entry.getRenameWhen()) && !expressionEvaluator.evaluateConditional(entry.getRenameWhen(), recordEvent)) { continue; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java index 87ff23247e..7d25bb456c 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java @@ -10,9 +10,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.annotations.AlsoRequired; import org.opensearch.dataprepper.model.annotations.ExampleValues; import org.opensearch.dataprepper.model.annotations.ExampleValues.Example; @@ -116,12 +118,31 @@ public Entry() { } } - @NotEmpty - @NotNull @Valid + @AlsoRequired(values = { + @AlsoRequired.Required(name = "transform_key", allowedValues = {"null", "none"}) + }) private List entries; + @JsonProperty(value = "transform_key", defaultValue = "none") + @JsonPropertyDescription("Allows transforming the key's name such as making the name all lowercase.") + @AlsoRequired(values = { + @AlsoRequired.Required(name = "entries", allowedValues = {"null"}) + }) + private TransformOption transformOption = TransformOption.NONE; + public List getEntries() { return entries; } + + public TransformOption getTransformOption() { + return transformOption; + } + + @AssertTrue(message = "entries and transform_key are mutually exclusive options. entries or transform_key is required.") + boolean isValidConfig() { + return (transformOption != null && transformOption != TransformOption.NONE) ^ entries != null; + } + + } diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java index d9d0c67dc4..8dba0d23cc 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java @@ -12,8 +12,11 @@ import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.common.TransformOption; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; +import org.opensearch.dataprepper.common.TransformOption; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -33,6 +36,8 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) @@ -59,6 +64,44 @@ void invalid_rename_when_throws_InvalidPluginConfigurationException() { assertThrows(InvalidPluginConfigurationException.class, this::createObjectUnderTest); } + @Test + void invalid_config_tests_with_entries_and_transform_options_both_present() throws NoSuchFieldException, IllegalAccessException { + final String renameWhen = UUID.randomUUID().toString(); + List entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen)); + + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.LOWERCASE); + assertFalse(renameKeyProcessorConfig.isValidConfig()); + } + + @Test + void invalid_config_tests_with_entries_and_transform_options_both_not_present() throws NoSuchFieldException, IllegalAccessException { + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); + assertFalse(renameKeyProcessorConfig.isValidConfig()); + } + + @Test + void invalid_config_tests_with_entries_and_transform_options_valid_entries() throws NoSuchFieldException, IllegalAccessException { + final String renameWhen = UUID.randomUUID().toString(); + List entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen)); + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE); + assertTrue(renameKeyProcessorConfig.isValidConfig()); + } + + @Test + void invalid_config_tests_with_entries_and_transform_options_valid_transformation() throws NoSuchFieldException, IllegalAccessException { + RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig(); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null); + ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.UPPERCASE); + assertTrue(renameKeyProcessorConfig.isValidConfig()); + } + @Test void invalid_config_when_both_from_key_empty_throws_InvalidPluginConfigurationException() { when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry(null,null, "newMessage", true, null))); @@ -224,7 +267,95 @@ public void testNoRename_when_RenameWhen_returns_false() { assertThat(editedRecords.get(0).getData().get("message", Object.class), equalTo("thisisamessage")); } + @Test + public void test_transformKey_converting_allkeys_lowercase() { + final String key1 = "KeY1"; + final String key2 = "kEy2"; + final String key3 = "keY3"; + final String key4 = "key4"; + final String key5 = "KEY5"; + Map data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555))); + when(mockConfig.getEntries()).thenReturn(null); + when(mockConfig.getTransformOption()).thenReturn(TransformOption.LOWERCASE); + Record record = buildRecordWithEvent(data); + final RenameKeyProcessor processor = createObjectUnderTest(); + final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key1), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false)); + + assertThat(editedRecords.get(0).getData().containsKey(key1.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key1.toLowerCase(), Integer.class), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key4.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key4.toLowerCase(), String.class), equalTo("value4")); + assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key5.toLowerCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key5.toLowerCase(), String.class), equalTo("5.555")); + } + + @Test + public void test_transformKey_converting_allkeys_uppercase() { + final String key1 = "KeY1"; + final String key2 = "kEy2"; + final String key3 = "keY3"; + final String key4 = "key4"; + final String key5 = "KEY5"; + Map data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555))); + when(mockConfig.getEntries()).thenReturn(null); + when(mockConfig.getTransformOption()).thenReturn(TransformOption.UPPERCASE); + Record record = buildRecordWithEvent(data); + final RenameKeyProcessor processor = createObjectUnderTest(); + final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key1), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false)); + + assertThat(editedRecords.get(0).getData().containsKey(key1.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key1.toUpperCase(), Integer.class), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key4.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key4.toUpperCase(), String.class), equalTo("value4")); + assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key5.toUpperCase()), is(true)); + assertThat(editedRecords.get(0).getData().get(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key5.toUpperCase(), String.class), equalTo("5.555")); + } + @Test + public void test_transformKey_converting_allkeys_capitalize() { + final String key1 = "key1"; + final String key2 = "key2"; + final String key3 = "key3"; + final String key4 = "key4"; + final String key5 = "key5"; + Map data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555))); + + when(mockConfig.getEntries()).thenReturn(null); + when(mockConfig.getTransformOption()).thenReturn(TransformOption.CAPITALIZE); + Record record = buildRecordWithEvent(data); + final RenameKeyProcessor processor = createObjectUnderTest(); + final List> editedRecords = (List>) processor.doExecute(Collections.singletonList(record)); + assertThat(editedRecords.size(), equalTo(1)); + assertThat(editedRecords.get(0).getData().containsKey(key1), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false)); + assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false)); + + assertThat(editedRecords.get(0).getData().containsKey("Key1"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3"), is(true)); + assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key4"), is(true)); + assertThat(editedRecords.get(0).getData().get("Key2/Key3/Key4", String.class), equalTo("value4")); + assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key5"), is(true)); + assertThat(editedRecords.get(0).getData().get("Key2/Key3/Key5", String.class), equalTo("5.555")); + } private RenameKeyProcessor createObjectUnderTest() { return new RenameKeyProcessor(pluginMetrics, mockConfig, expressionEvaluator);