Skip to content

Commit 09d6495

Browse files
kkondakakkondaka
committed
Add support for convering keys to lowercase/uppercase in RenameKeyProcessor
Signed-off-by: Krishna Kondaka <krishkdk@amazon.com>
1 parent 4dab7ac commit 09d6495

8 files changed

Lines changed: 100 additions & 6 deletions

File tree

data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOption.java renamed to data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/common/TransformOption.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
package org.opensearch.dataprepper.plugins.processor.keyvalue;
6+
package org.opensearch.dataprepper.common;
77

88
import com.fasterxml.jackson.annotation.JsonCreator;
99
import com.fasterxml.jackson.annotation.JsonValue;
@@ -39,7 +39,7 @@ public String getTransformName() {
3939
return transformName;
4040
}
4141

42-
Function<String, String> getTransformFunction() {
42+
public Function<String, String> getTransformFunction() {
4343
return transformFunction;
4444
}
4545

data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/TransformOptionTest.java renamed to data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/common/TransformOptionTest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
package org.opensearch.dataprepper.plugins.processor.keyvalue;
6+
package org.opensearch.dataprepper.common;
77

88
import org.junit.jupiter.api.Test;
99
import org.junit.jupiter.api.extension.ExtensionContext;
@@ -88,4 +88,4 @@ public Stream<? extends Arguments> provideArguments(final ExtensionContext exten
8888
);
8989
}
9090
}
91-
}
91+
}

‎data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessor.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
1111
import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
1212
import org.opensearch.dataprepper.model.event.Event;
13+
import org.opensearch.dataprepper.common.TransformOption;
1314
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
1415
import org.opensearch.dataprepper.model.processor.AbstractProcessor;
1516
import org.opensearch.dataprepper.model.processor.Processor;

‎data-prepper-plugins/key-value-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import jakarta.validation.constraints.NotNull;
1414
import jakarta.validation.constraints.AssertTrue;
1515
import jakarta.validation.constraints.Size;
16+
import org.opensearch.dataprepper.common.TransformOption;
1617
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
1718
import org.opensearch.dataprepper.model.annotations.ExampleValues;
1819
import org.opensearch.dataprepper.model.annotations.ExampleValues.Example;

‎data-prepper-plugins/key-value-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.mockito.Mock;
1717
import org.mockito.junit.jupiter.MockitoExtension;
1818
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
19+
import org.opensearch.dataprepper.common.TransformOption;
1920
import org.opensearch.dataprepper.metrics.PluginMetrics;
2021
import org.opensearch.dataprepper.model.event.Event;
2122
import org.opensearch.dataprepper.model.event.JacksonEvent;

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.opensearch.dataprepper.metrics.PluginMetrics;
1212
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
1313
import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
14+
import org.opensearch.dataprepper.common.TransformOption;
1415
import org.opensearch.dataprepper.model.event.Event;
1516
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
1617
import org.opensearch.dataprepper.model.processor.AbstractProcessor;
@@ -32,13 +33,23 @@ public class RenameKeyProcessor extends AbstractProcessor<Record<Event>, Record<
3233
private final List<RenameKeyProcessorConfig.Entry> entries;
3334

3435
private final ExpressionEvaluator expressionEvaluator;
36+
private final TransformOption transformOption;
3537

3638
@DataPrepperPluginConstructor
3739
public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProcessorConfig config, final ExpressionEvaluator expressionEvaluator) {
3840
super(pluginMetrics);
3941
this.entries = config.getEntries();
4042
this.expressionEvaluator = expressionEvaluator;
43+
this.transformOption = config.getTransformOption();
4144

45+
boolean entriesProvided = config.getEntries() != null;
46+
boolean transformOptionProvided = (transformOption != null && transformOption != TransformOption.NONE);
47+
if ((!entriesProvided ^ transformOptionProvided)) {
48+
throw new InvalidPluginConfigurationException("Only one of 'entries' or 'transform_key' should be specified.");
49+
}
50+
if (config.getEntries() == null || config.getEntries().size() == 0) {
51+
return;
52+
}
4253
config.getEntries().forEach(entry -> {
4354
if (entry.getRenameWhen() != null
4455
&& !expressionEvaluator.isValidExpressionStatement(entry.getRenameWhen())) {
@@ -55,13 +66,32 @@ public RenameKeyProcessor(final PluginMetrics pluginMetrics, final RenameKeyProc
5566
});
5667
}
5768

69+
private void transformEvent(final Event event, Map<String, Object> map, final String keyPrefix) {
70+
for (Map.Entry<String, Object> entry : map.entrySet()) {
71+
Object result = null;
72+
try {
73+
if (entry.getValue() instanceof Map) {
74+
transformEvent(event, (Map<String, Object>)entry.getValue(), keyPrefix+entry.getKey()+"/");
75+
}
76+
Object value = event.get(keyPrefix+entry.getKey(), Object.class);
77+
event.delete(keyPrefix+entry.getKey());
78+
event.put(keyPrefix+transformOption.getTransformFunction().apply(entry.getKey()), value);
79+
} catch (Exception ignored) {}
80+
}
81+
}
82+
5883
@Override
5984
public Collection<Record<Event>> doExecute(final Collection<Record<Event>> records) {
6085
for(final Record<Event> record : records) {
6186
final Event recordEvent = record.getData();
6287

6388
try {
6489

90+
if (transformOption != null && transformOption != TransformOption.NONE) {
91+
transformEvent(recordEvent, recordEvent.toMap(), "");
92+
continue;
93+
}
94+
6595
for (RenameKeyProcessorConfig.Entry entry : entries) {
6696
if (Objects.nonNull(entry.getRenameWhen()) && !expressionEvaluator.evaluateConditional(entry.getRenameWhen(), recordEvent)) {
6797
continue;

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorConfig.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import jakarta.validation.Valid;
1414
import jakarta.validation.constraints.NotEmpty;
1515
import jakarta.validation.constraints.NotNull;
16+
import org.opensearch.dataprepper.common.TransformOption;
1617
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
1718
import org.opensearch.dataprepper.model.annotations.ExampleValues;
1819
import org.opensearch.dataprepper.model.annotations.ExampleValues.Example;
@@ -116,12 +117,25 @@ public Entry() {
116117
}
117118
}
118119

119-
@NotEmpty
120-
@NotNull
121120
@Valid
121+
@AlsoRequired(values = {
122+
@AlsoRequired.Required(name = "transform_key", allowedValues = {"null", "none"})
123+
})
122124
private List<Entry> entries;
123125

126+
@JsonProperty(value = "transform_key", defaultValue = "none")
127+
@JsonPropertyDescription("Allows transforming the key's name such as making the name all lowercase.")
128+
@AlsoRequired(values = {
129+
@AlsoRequired.Required(name = "entries", allowedValues = {"null"})
130+
})
131+
private TransformOption transformOption = TransformOption.NONE;
132+
124133
public List<Entry> getEntries() {
125134
return entries;
126135
}
136+
137+
public TransformOption getTransformOption() {
138+
return transformOption;
139+
}
140+
127141
}

‎data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessorTests.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.opensearch.dataprepper.model.event.EventKey;
1313
import org.opensearch.dataprepper.model.event.EventKeyFactory;
1414
import org.opensearch.dataprepper.model.event.JacksonEvent;
15+
import org.opensearch.dataprepper.common.TransformOption;
1516
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
1617
import org.opensearch.dataprepper.model.record.Record;
1718
import org.junit.jupiter.api.Test;
@@ -224,7 +225,53 @@ public void testNoRename_when_RenameWhen_returns_false() {
224225
assertThat(editedRecords.get(0).getData().get("message", Object.class), equalTo("thisisamessage"));
225226
}
226227

228+
@Test
229+
public void test_transformKey_converting_allkeys_lowercase() {
230+
Map<String, Object> data = Map.of("KeY1", 1, "kEy2", Map.of("keY3", Map.of("key4", "value4", "KEY5", 5.555)));
231+
when(mockConfig.getEntries()).thenReturn(null);
232+
when(mockConfig.getTransformOption()).thenReturn(TransformOption.LOWERCASE);
233+
Record<Event> record = buildRecordWithEvent(data);
234+
final RenameKeyProcessor processor = createObjectUnderTest();
235+
final List<Record<Event>> editedRecords = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
236+
assertThat(editedRecords.size(), equalTo(1));
237+
assertThat(editedRecords.get(0).getData().containsKey("key1"), is(true));
238+
assertThat(editedRecords.get(0).getData().containsKey("key2"), is(true));
239+
assertThat(editedRecords.get(0).getData().containsKey("key2/key3"), is(true));
240+
assertThat(editedRecords.get(0).getData().containsKey("key2/key3/key4"), is(true));
241+
assertThat(editedRecords.get(0).getData().containsKey("key2/key3/key5"), is(true));
242+
}
227243

244+
@Test
245+
public void test_transformKey_converting_allkeys_uppercase() {
246+
Map<String, Object> data = Map.of("KeY1", 1, "kEy2", Map.of("keY3", Map.of("key4", "value4", "KEY5", 5.555)));
247+
when(mockConfig.getEntries()).thenReturn(null);
248+
when(mockConfig.getTransformOption()).thenReturn(TransformOption.UPPERCASE);
249+
Record<Event> record = buildRecordWithEvent(data);
250+
final RenameKeyProcessor processor = createObjectUnderTest();
251+
final List<Record<Event>> editedRecords = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
252+
assertThat(editedRecords.size(), equalTo(1));
253+
assertThat(editedRecords.get(0).getData().containsKey("KEY1"), is(true));
254+
assertThat(editedRecords.get(0).getData().containsKey("KEY2"), is(true));
255+
assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3"), is(true));
256+
assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3/KEY4"), is(true));
257+
assertThat(editedRecords.get(0).getData().containsKey("KEY2/KEY3/KEY5"), is(true));
258+
}
259+
260+
@Test
261+
public void test_transformKey_converting_allkeys_capitalize() {
262+
Map<String, Object> data = Map.of("key1", 1, "key2", Map.of("key3", Map.of("key4", "value4", "Key5", 5.555)));
263+
when(mockConfig.getEntries()).thenReturn(null);
264+
when(mockConfig.getTransformOption()).thenReturn(TransformOption.CAPITALIZE);
265+
Record<Event> record = buildRecordWithEvent(data);
266+
final RenameKeyProcessor processor = createObjectUnderTest();
267+
final List<Record<Event>> editedRecords = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
268+
assertThat(editedRecords.size(), equalTo(1));
269+
assertThat(editedRecords.get(0).getData().containsKey("Key1"), is(true));
270+
assertThat(editedRecords.get(0).getData().containsKey("Key2"), is(true));
271+
assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3"), is(true));
272+
assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key4"), is(true));
273+
assertThat(editedRecords.get(0).getData().containsKey("Key2/Key3/Key5"), is(true));
274+
}
228275

229276
private RenameKeyProcessor createObjectUnderTest() {
230277
return new RenameKeyProcessor(pluginMetrics, mockConfig, expressionEvaluator);

0 commit comments

Comments
 (0)