diff --git a/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunction.java b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunction.java new file mode 100644 index 0000000000..987baad3ae --- /dev/null +++ b/data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunction.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.expression; + +import org.opensearch.dataprepper.model.event.Event; + +import javax.inject.Named; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; + +/** + * Expression function that generates a random UUID (version 4) string. + * Usage: {@code generateUuid()} + */ +@Named +public class GenerateUuidExpressionFunction implements ExpressionFunction { + + static final String FUNCTION_NAME = "generateUuid"; + + @Override + public String getFunctionName() { + return FUNCTION_NAME; + } + + @Override + public Object evaluate(final List args, final Event event, final Function convertLiteralType) { + if (!args.isEmpty()) { + throw new RuntimeException(FUNCTION_NAME + "() does not take any arguments"); + } + return UUID.randomUUID().toString(); + } +} diff --git a/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunctionTest.java b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunctionTest.java new file mode 100644 index 0000000000..edcca1bf5f --- /dev/null +++ b/data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunctionTest.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.expression; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.event.Event; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; + +class GenerateUuidExpressionFunctionTest { + + private final GenerateUuidExpressionFunction function = new GenerateUuidExpressionFunction(); + private final Event event = mock(Event.class); + private final Function convertLiteralType = v -> v; + + @Test + void getFunctionName_returns_generateUuid() { + assertThat(function.getFunctionName(), equalTo("generateUuid")); + } + + @Test + void evaluate_returns_valid_uuid_string() { + final Object result = function.evaluate(Collections.emptyList(), event, convertLiteralType); + assertThat(result, instanceOf(String.class)); + final String uuidStr = (String) result; + final UUID regeneratedUuid = assertDoesNotThrow(() -> UUID.fromString(uuidStr)); + assertThat(regeneratedUuid.toString(), equalTo(uuidStr)); + } + + @Test + void evaluate_returns_unique_values_on_successive_calls() { + final String first = (String) function.evaluate(Collections.emptyList(), event, convertLiteralType); + final String second = (String) function.evaluate(Collections.emptyList(), event, convertLiteralType); + assertThat(first, not(equalTo(second))); + } + + @Test + void evaluate_throws_when_args_are_provided() { + assertThrows(RuntimeException.class, + () -> function.evaluate(List.of("unexpected"), event, convertLiteralType)); + } +} diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java index 0908e072d2..77de3bf5c3 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java @@ -313,7 +313,7 @@ private Object retrieveValue(final AddEntryProcessorConfig.Entry entry, final Ev int entryIndex = entries.indexOf(entry); EntryProperties props = entryProperties.get(entryIndex); KeyInfo keyInfo = preprocessedKeys.get(entryIndex); - + if (!Objects.isNull(entry.getValueExpression())) { value = expressionEvaluator.evaluate(entry.getValueExpression(), context); } else if (!Objects.isNull(entry.getFormat())) { diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java index 8456ff90eb..9e8af89643 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java @@ -214,7 +214,7 @@ public boolean getFlattenKey(){ public String getAddWhen() { return addWhen; } - @AssertTrue(message = "Either value or format or expression must be specified, and only one of them can be specified") + @AssertTrue(message = "Exactly one of value, format, or value_expression must be specified") public boolean hasValueOrFormatOrExpression() { return Stream.of(value, format, valueExpression).filter(n -> n!=null).count() == 1; } @@ -250,7 +250,6 @@ public Entry(final String key, if (metadataKey != null && iterateOn != null) { throw new IllegalArgumentException("iterate_on cannot be applied to metadata"); } - if (iterateOn == null && addToElementWhen != null) { throw new InvalidPluginConfigurationException("add_to_element_when only applies when iterate_on is configured."); } diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java index 07d29bf8b0..17395cb33a 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java @@ -1088,6 +1088,96 @@ public void testAddFlattenedNestedEntryIterateOn() { equalTo(List.of(Map.of("key", 5, "nested/newMessage", 3)))); // [{"key": 5, "nested/newMessage": 3}}] } + @Test + void test_generateUuid_expression_adds_uuid_string_to_event() { + final String uuidExpr = "generateUuid()"; + final String generatedUuid = UUID.randomUUID().toString(); + when(mockConfig.getEntries()).thenReturn(createListOfEntries( + createEntry("recordId", null, null, null, uuidExpr, false, false, null, null, null))); + when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true); + when(expressionEvaluator.evaluate(eq(uuidExpr), any())).thenReturn(generatedUuid); + + final AddEntryProcessor processor = createObjectUnderTest(); + final Record record = getEvent("test-message"); + final List> result = (List>) processor.doExecute(Collections.singletonList(record)); + + final Event event = result.get(0).getData(); + assertThat(event.containsKey("recordId"), is(true)); + assertThat(event.get("recordId", String.class), equalTo(generatedUuid)); + } + + @Test + void test_generateUuid_expression_produces_unique_values_per_event() { + final String uuidExpr = "generateUuid()"; + final String uuid1 = UUID.randomUUID().toString(); + final String uuid2 = UUID.randomUUID().toString(); + when(mockConfig.getEntries()).thenReturn(createListOfEntries( + createEntry("recordId", null, null, null, uuidExpr, false, false, null, null, null))); + when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true); + when(expressionEvaluator.evaluate(eq(uuidExpr), any())) + .thenReturn(uuid1) + .thenReturn(uuid2); + + final AddEntryProcessor processor = createObjectUnderTest(); + final List> result = (List>) processor.doExecute( + Arrays.asList(getEvent("message-one"), getEvent("message-two"))); + + assertThat(result.get(0).getData().get("recordId", String.class), equalTo(uuid1)); + assertThat(result.get(1).getData().get("recordId", String.class), equalTo(uuid2)); + assertThat(uuid1.equals(uuid2), is(false)); + } + + @Test + void test_generateUuid_expression_does_not_overwrite_existing_key_by_default() { + final String uuidExpr = "generateUuid()"; + when(mockConfig.getEntries()).thenReturn(createListOfEntries( + createEntry("existingId", null, null, null, uuidExpr, false, false, null, null, null))); + when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true); + + final AddEntryProcessor processor = createObjectUnderTest(); + final Map data = new HashMap<>(); + data.put("existingId", "original-value"); + final Record record = buildRecordWithEvent(data); + final List> result = (List>) processor.doExecute(Collections.singletonList(record)); + + assertThat(result.get(0).getData().get("existingId", String.class), equalTo("original-value")); + } + + @Test + void test_generateUuid_expression_overwrites_existing_key_when_overwrite_is_true() { + final String uuidExpr = "generateUuid()"; + final String newUuid = UUID.randomUUID().toString(); + when(mockConfig.getEntries()).thenReturn(createListOfEntries( + createEntry("existingId", null, null, null, uuidExpr, true, false, null, null, null))); + when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true); + when(expressionEvaluator.evaluate(eq(uuidExpr), any())).thenReturn(newUuid); + + final AddEntryProcessor processor = createObjectUnderTest(); + final Map data = new HashMap<>(); + data.put("existingId", "original-value"); + final Record record = buildRecordWithEvent(data); + final List> result = (List>) processor.doExecute(Collections.singletonList(record)); + + assertThat(result.get(0).getData().get("existingId", String.class), equalTo(newUuid)); + } + + @Test + void test_generateUuid_expression_respects_add_when_condition() { + final String uuidExpr = "generateUuid()"; + final String addWhen = "/skip == true"; + when(mockConfig.getEntries()).thenReturn(createListOfEntries( + createEntry("recordId", null, null, null, uuidExpr, false, false, addWhen, null, null))); + when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true); + when(expressionEvaluator.isValidExpressionStatement(addWhen)).thenReturn(true); + when(expressionEvaluator.evaluateConditional(eq(addWhen), any())).thenReturn(false); + + final AddEntryProcessor processor = createObjectUnderTest(); + final Record record = getEvent("message"); + final List> result = (List>) processor.doExecute(Collections.singletonList(record)); + + assertThat(result.get(0).getData().containsKey("recordId"), is(false)); + } + private AddEntryProcessor createObjectUnderTest() { return new AddEntryProcessor(pluginMetrics, mockConfig, expressionEvaluator, eventKeyFactory); } @@ -1124,6 +1214,7 @@ private AddEntryProcessorConfig.Entry createEntry( iterateOn, addToElementWhen); } + private List createListOfEntries(final AddEntryProcessorConfig.Entry... entries) { return new LinkedList<>(Arrays.asList(entries)); }