Skip to content

Add support for convering keys to lowercase/uppercase in RenameKeyProcessor #5810

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kkondaka merged 3 commits into from
Jun 24, 2025
Merged

Add support for convering keys to lowercase/uppercase in RenameKeyProcessor #5810

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
09d6495
Add support for convering keys to lowercase/uppercase in RenameKeyPro…
kkondaka Jun 23, 2025
d5b740e
Addressed review comments
kkondaka Jun 24, 2025
578bc49
Addressed review comments
kkondaka Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...s/processor/keyvalue/TransformOption.java → ...h/dataprepper/common/TransformOption.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugins.processor.keyvalue;
package org.opensearch.dataprepper.common;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
Expand Down Expand Up @@ -39,7 +39,7 @@ public String getTransformName() {
return transformName;
}

Function<String, String> getTransformFunction() {
public Function<String, String> getTransformFunction() {
return transformFunction;
}

Expand Down
4 changes: 2 additions & 2 deletions ...ocessor/keyvalue/TransformOptionTest.java → ...taprepper/common/TransformOptionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.dataprepper.plugins.processor.keyvalue;
package org.opensearch.dataprepper.common;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtensionContext;
Expand Down Expand Up @@ -88,4 +88,4 @@ public Stream<? extends Arguments> provideArguments(final ExtensionContext exten
);
}
}
}
}
1 change: 1 addition & 0 deletions ...rc/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.common.TransformOption;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
import org.opensearch.dataprepper.model.processor.AbstractProcessor;
import org.opensearch.dataprepper.model.processor.Processor;
Expand Down
1 change: 1 addition & 0 deletions ...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.AssertTrue;
import jakarta.validation.constraints.Size;
import org.opensearch.dataprepper.common.TransformOption;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ExampleValues;
import org.opensearch.dataprepper.model.annotations.ExampleValues.Example;
Expand Down
1 change: 1 addition & 0 deletions ...st/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
import org.opensearch.dataprepper.common.TransformOption;
import org.opensearch.dataprepper.metrics.PluginMetrics;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.event.JacksonEvent;
Expand Down
3 changes: 2 additions & 1 deletion data-prepper-plugins/mutate-event-processors/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,5 @@ dependencies {
testImplementation project(':data-prepper-test:test-event')
testImplementation testLibs.slf4j.simple
testImplementation testLibs.spring.test
}
testImplementation project(':data-prepper-test:test-common')
}
43 changes: 33 additions & 10 deletions ...ain/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.opensearch.dataprepper.metrics.PluginMetrics;
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
import org.opensearch.dataprepper.common.TransformOption;
import org.opensearch.dataprepper.model.event.Event;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
import org.opensearch.dataprepper.model.processor.AbstractProcessor;
Expand All @@ -32,27 +33,44 @@ public class RenameKeyProcessor extends AbstractProcessor<Record<Event>, Record<
private final List<RenameKeyProcessorConfig.Entry> entries;

private final ExpressionEvaluator expressionEvaluator;
private final TransformOption transformOption;

@DataPrepperPluginConstructor
public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProcessorConfig config, final ExpressionEvaluator expressionEvaluator) {
super(pluginMetrics);
this.entries = config.getEntries();
this.expressionEvaluator = expressionEvaluator;
this.transformOption = config.getTransformOption();

config.getEntries().forEach(entry -> {
if (entry.getRenameWhen() != null
if (config.getEntries() != null) {
config.getEntries().forEach(entry -> {
if (entry.getRenameWhen() != null
&& !expressionEvaluator.isValidExpressionStatement(entry.getRenameWhen())) {
throw new InvalidPluginConfigurationException(
String.format("rename_when %s is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax",
entry.getRenameWhen()));
}
if (entry.getFromKey() == null && entry.getFromKeyPattern() == null) {
throw new InvalidPluginConfigurationException("Either from_key or from_key_regex must be specified. ");
}
if (entry.getFromKey() != null && entry.getFromKeyPattern() != null) {
throw new InvalidPluginConfigurationException("Only one of from_key or from_key_regex should be specified.");
}
});
}
if (entry.getFromKey() == null && entry.getFromKeyPattern() == null) {
throw new InvalidPluginConfigurationException("Either from_key or from_key_regex must be specified. ");
}
if (entry.getFromKey() != null && entry.getFromKeyPattern() != null) {
throw new InvalidPluginConfigurationException("Only one of from_key or from_key_regex should be specified.");
}
});
}
}

private void transformEvent(final Event event, Map<String, Object> map, final String keyPrefix) {
for (Map.Entry<String, Object> entry : map.entrySet()) {
try {
if (entry.getValue() instanceof Map) {
transformEvent(event, (Map<String, Object>)entry.getValue(), keyPrefix+entry.getKey()+"/");
}
Object value = event.get(keyPrefix+entry.getKey(), Object.class);
event.delete(keyPrefix+entry.getKey());
event.put(keyPrefix+transformOption.getTransformFunction().apply(entry.getKey()), value);
} catch (Exception ignored) {}
}
}

@Override
Expand All @@ -62,6 +80,11 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor

try {

if (transformOption != null && transformOption != TransformOption.NONE) {
transformEvent(recordEvent, recordEvent.toMap(), "");
continue;
}

for (RenameKeyProcessorConfig.Entry entry : entries) {
if (Objects.nonNull(entry.getRenameWhen()) && !expressionEvaluator.evaluateConditional(entry.getRenameWhen(), recordEvent)) {
continue;
Expand Down
25 changes: 23 additions & 2 deletions ...va/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import jakarta.validation.constraints.AssertTrue;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import org.opensearch.dataprepper.common.TransformOption;
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
import org.opensearch.dataprepper.model.annotations.ExampleValues;
import org.opensearch.dataprepper.model.annotations.ExampleValues.Example;
Expand Down Expand Up @@ -116,12 +118,31 @@ public Entry() {
}
}

@NotEmpty
@NotNull
@Valid
@AlsoRequired(values = {
@AlsoRequired.Required(name = "transform_key", allowedValues = {"null", "none"})
})
private List<Entry> entries;

@JsonProperty(value = "transform_key", defaultValue = "none")
@JsonPropertyDescription("Allows transforming the key's name such as making the name all lowercase.")
@AlsoRequired(values = {
@AlsoRequired.Required(name = "entries", allowedValues = {"null"})
})
private TransformOption transformOption = TransformOption.NONE;

public List<Entry> getEntries() {
return entries;
}

public TransformOption getTransformOption() {
return transformOption;
}

@AssertTrue(message = "entries and transform_key are mutually exclusive options. entries or transform_key is required.")
boolean isValidConfig() {
return (transformOption != null && transformOption != TransformOption.NONE) ^ entries != null;
}


}
131 changes: 131 additions & 0 deletions ...ava/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java
View file
Open in desktop
Copy link
Copy Markdown
Collaborator

@san81 san81 Jun 23, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we have one more test case to validate the logic in (!entriesProvided ^ transformOptionProvided) Like to test this validation part

Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@
import org.opensearch.dataprepper.model.event.EventKey;
import org.opensearch.dataprepper.model.event.EventKeyFactory;
import org.opensearch.dataprepper.model.event.JacksonEvent;
import org.opensearch.dataprepper.common.TransformOption;
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
import org.opensearch.dataprepper.model.record.Record;
import org.opensearch.dataprepper.test.helper.ReflectivelySetField;
import org.opensearch.dataprepper.common.TransformOption;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
Expand All @@ -33,6 +36,8 @@
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
Expand All @@ -59,6 +64,44 @@ void invalid_rename_when_throws_InvalidPluginConfigurationException() {
assertThrows(InvalidPluginConfigurationException.class, this::createObjectUnderTest);
}

@Test
void invalid_config_tests_with_entries_and_transform_options_both_present() throws NoSuchFieldException, IllegalAccessException {
final String renameWhen = UUID.randomUUID().toString();
List<RenameKeyProcessorConfig.Entry> entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen));

RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig();

ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries);
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.LOWERCASE);
assertFalse(renameKeyProcessorConfig.isValidConfig());
Copy link
Copy Markdown
Member

@dlvenable dlvenable Jun 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These should all be separate @Test methods because they are different tests.

}

@Test
void invalid_config_tests_with_entries_and_transform_options_both_not_present() throws NoSuchFieldException, IllegalAccessException {
RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig();
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null);
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE);
assertFalse(renameKeyProcessorConfig.isValidConfig());
}

@Test
void invalid_config_tests_with_entries_and_transform_options_valid_entries() throws NoSuchFieldException, IllegalAccessException {
final String renameWhen = UUID.randomUUID().toString();
List<RenameKeyProcessorConfig.Entry> entries = createListOfEntries(createEntry("message", null,"newMessage", true, renameWhen));
RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig();
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", entries);
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.NONE);
assertTrue(renameKeyProcessorConfig.isValidConfig());
}

@Test
void invalid_config_tests_with_entries_and_transform_options_valid_transformation() throws NoSuchFieldException, IllegalAccessException {
RenameKeyProcessorConfig renameKeyProcessorConfig = new RenameKeyProcessorConfig();
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "entries", null);
ReflectivelySetField.setField(RenameKeyProcessorConfig.class, renameKeyProcessorConfig, "transformOption", TransformOption.UPPERCASE);
assertTrue(renameKeyProcessorConfig.isValidConfig());
}

@Test
void invalid_config_when_both_from_key_empty_throws_InvalidPluginConfigurationException() {
when(mockConfig.getEntries()).thenReturn(createListOfEntries(createEntry(null,null, "newMessage", true, null)));
Expand Down Expand Up @@ -224,7 +267,95 @@ public void testNoRename_when_RenameWhen_returns_false() {
assertThat(editedRecords.get(0).getData().get("message", Object.class), equalTo("thisisamessage"));
}

@Test
public void test_transformKey_converting_allkeys_lowercase() {
final String key1 = "KeY1";
final String key2 = "kEy2";
final String key3 = "keY3";
final String key4 = "key4";
final String key5 = "KEY5";
Map<String, Object> data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555)));
when(mockConfig.getEntries()).thenReturn(null);
when(mockConfig.getTransformOption()).thenReturn(TransformOption.LOWERCASE);
Record<Event> record = buildRecordWithEvent(data);
final RenameKeyProcessor processor = createObjectUnderTest();
final List<Record<Event>> editedRecords = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
assertThat(editedRecords.size(), equalTo(1));
assertThat(editedRecords.get(0).getData().containsKey(key1), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false));

assertThat(editedRecords.get(0).getData().containsKey(key1.toLowerCase()), is(true));
assertThat(editedRecords.get(0).getData().get(key1.toLowerCase(), Integer.class), equalTo(1));
assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()), is(true));
assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()), is(true));
assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key4.toLowerCase()), is(true));
assertThat(editedRecords.get(0).getData().get(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key4.toLowerCase(), String.class), equalTo("value4"));
assertThat(editedRecords.get(0).getData().containsKey(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key5.toLowerCase()), is(true));
assertThat(editedRecords.get(0).getData().get(key2.toLowerCase()+"/"+key3.toLowerCase()+"/"+key5.toLowerCase(), String.class), equalTo("5.555"));
}

@Test
public void test_transformKey_converting_allkeys_uppercase() {
final String key1 = "KeY1";
final String key2 = "kEy2";
final String key3 = "keY3";
final String key4 = "key4";
final String key5 = "KEY5";
Map<String, Object> data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555)));
when(mockConfig.getEntries()).thenReturn(null);
when(mockConfig.getTransformOption()).thenReturn(TransformOption.UPPERCASE);
Record<Event> record = buildRecordWithEvent(data);
final RenameKeyProcessor processor = createObjectUnderTest();
final List<Record<Event>> editedRecords = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
assertThat(editedRecords.size(), equalTo(1));
assertThat(editedRecords.get(0).getData().containsKey(key1), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false));

assertThat(editedRecords.get(0).getData().containsKey(key1.toUpperCase()), is(true));
assertThat(editedRecords.get(0).getData().get(key1.toUpperCase(), Integer.class), equalTo(1));
assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()), is(true));
assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()), is(true));
assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key4.toUpperCase()), is(true));
assertThat(editedRecords.get(0).getData().get(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key4.toUpperCase(), String.class), equalTo("value4"));
assertThat(editedRecords.get(0).getData().containsKey(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key5.toUpperCase()), is(true));
assertThat(editedRecords.get(0).getData().get(key2.toUpperCase()+"/"+key3.toUpperCase()+"/"+key5.toUpperCase(), String.class), equalTo("5.555"));
}

@Test
public void test_transformKey_converting_allkeys_capitalize() {
final String key1 = "key1";
final String key2 = "key2";
final String key3 = "key3";
final String key4 = "key4";
final String key5 = "key5";
Map<String, Object> data = Map.of(key1, 1, key2, Map.of(key3, Map.of(key4, "value4", key5, 5.555)));

when(mockConfig.getEntries()).thenReturn(null);
when(mockConfig.getTransformOption()).thenReturn(TransformOption.CAPITALIZE);
Record<Event> record = buildRecordWithEvent(data);
final RenameKeyProcessor processor = createObjectUnderTest();
final List<Record<Event>> editedRecords = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
assertThat(editedRecords.size(), equalTo(1));
assertThat(editedRecords.get(0).getData().containsKey(key1), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key4), is(false));
assertThat(editedRecords.get(0).getData().containsKey(key2+"/"+key3+"/"+key5), is(false));

assertThat(editedRecords.get(0).getData().containsKey("Key1"), is(true));
assertThat(editedRecords.get(0).getData().containsKey("Key2"), is(true));
assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3"), is(true));
assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key4"), is(true));
assertThat(editedRecords.get(0).getData().get("Key2/Key3/Key4", String.class), equalTo("value4"));
assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key5"), is(true));
assertThat(editedRecords.get(0).getData().get("Key2/Key3/Key5", String.class), equalTo("5.555"));
}

private RenameKeyProcessor createObjectUnderTest() {
return new RenameKeyProcessor(pluginMetrics, mockConfig, expressionEvaluator);
Expand Down
Loading