Skip to content

Commit a0960bd

Browse files
ZhangxunmtZhangxunmt
committed
use expression function for uuid creation
Signed-off-by: Xun Zhang <xunzh@amazon.com>
1 parent f5e341f commit a0960bd

5 files changed

Lines changed: 137 additions & 88 deletions

File tree

‎data-prepper-expression/src/main/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunction.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.expression;
11+
12+
import org.opensearch.dataprepper.model.event.Event;
13+
14+
import javax.inject.Named;
15+
import java.util.List;
16+
import java.util.UUID;
17+
import java.util.function.Function;
18+
19+
/**
20+
* Expression function that generates a random UUID (version 4) string.
21+
* Usage: {@code generate_uuid()}
22+
*/
23+
@Named
24+
public class GenerateUuidExpressionFunction implements ExpressionFunction {
25+
26+
static final String FUNCTION_NAME = "generate_uuid";
27+
28+
@Override
29+
public String getFunctionName() {
30+
return FUNCTION_NAME;
31+
}
32+
33+
@Override
34+
public Object evaluate(final List<Object> args, final Event event, final Function<Object, Object> convertLiteralType) {
35+
if (!args.isEmpty()) {
36+
throw new RuntimeException(FUNCTION_NAME + "() does not take any arguments");
37+
}
38+
return UUID.randomUUID().toString();
39+
}
40+
}

‎data-prepper-expression/src/test/java/org/opensearch/dataprepper/expression/GenerateUuidExpressionFunctionTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.expression;
11+
12+
import org.junit.jupiter.api.Test;
13+
import org.opensearch.dataprepper.model.event.Event;
14+
15+
import java.util.Collections;
16+
import java.util.List;
17+
import java.util.UUID;
18+
import java.util.function.Function;
19+
20+
import static org.hamcrest.CoreMatchers.instanceOf;
21+
import static org.hamcrest.CoreMatchers.not;
22+
import static org.hamcrest.CoreMatchers.equalTo;
23+
import static org.hamcrest.MatcherAssert.assertThat;
24+
import static org.junit.jupiter.api.Assertions.assertThrows;
25+
import static org.mockito.Mockito.mock;
26+
27+
class GenerateUuidExpressionFunctionTest {
28+
29+
private final GenerateUuidExpressionFunction function = new GenerateUuidExpressionFunction();
30+
private final Event event = mock(Event.class);
31+
private final Function<Object, Object> convertLiteralType = v -> v;
32+
33+
@Test
34+
void getFunctionName_returns_generate_uuid() {
35+
assertThat(function.getFunctionName(), equalTo("generate_uuid"));
36+
}
37+
38+
@Test
39+
void evaluate_returns_valid_uuid_string() {
40+
final Object result = function.evaluate(Collections.emptyList(), event, convertLiteralType);
41+
assertThat(result, instanceOf(String.class));
42+
final String uuidStr = (String) result;
43+
// UUID.fromString throws if invalid
44+
assertThat(UUID.fromString(uuidStr).toString(), equalTo(uuidStr));
45+
}
46+
47+
@Test
48+
void evaluate_returns_unique_values_on_successive_calls() {
49+
final String first = (String) function.evaluate(Collections.emptyList(), event, convertLiteralType);
50+
final String second = (String) function.evaluate(Collections.emptyList(), event, convertLiteralType);
51+
assertThat(first, not(equalTo(second)));
52+
}
53+
54+
@Test
55+
void evaluate_throws_when_args_are_provided() {
56+
assertThrows(RuntimeException.class,
57+
() -> function.evaluate(List.of("unexpected"), event, convertLiteralType));
58+
}
59+
}

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import java.util.List;
3131
import java.util.Map;
3232
import java.util.Objects;
33-
import java.util.UUID;
3433
import java.util.function.Consumer;
3534
import java.util.function.Supplier;
3635

@@ -315,9 +314,7 @@ private Object retrieveValue(final AddEntryProcessorConfig.Entry entry, final Ev
315314
EntryProperties props = entryProperties.get(entryIndex);
316315
KeyInfo keyInfo = preprocessedKeys.get(entryIndex);
317316

318-
if (entry.getGenerateUuid()) {
319-
return UUID.randomUUID().toString();
320-
} else if (!Objects.isNull(entry.getValueExpression())) {
317+
if (!Objects.isNull(entry.getValueExpression())) {
321318
value = expressionEvaluator.evaluate(entry.getValueExpression(), context);
322319
} else if (!Objects.isNull(entry.getFormat())) {
323320
try {

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorConfig.java‎

Lines changed: 2 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -166,12 +166,6 @@ public static class Entry {
166166
})
167167
private boolean flattenKey = true;
168168

169-
@JsonProperty("generate_uuid")
170-
@JsonPropertyDescription("When set to <code>true</code>, generates a random UUID (version 4) as the value of the new entry. " +
171-
"Each event receives its own unique UUID, providing globally unique identifiers across distributed deployments " +
172-
"without any coordination between nodes. Cannot be used together with <code>value</code>, <code>format</code>, or <code>value_expression</code>.")
173-
private boolean generateUuid = false;
174-
175169
@JsonProperty("add_when")
176170
@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, " +
177171
"such as <code>/some-key == \"test\"</code>, that will be evaluated to determine whether the processor will be run on the event.")
@@ -218,19 +212,11 @@ public boolean getFlattenKey(){
218212
return flattenKey;
219213
}
220214

221-
public boolean getGenerateUuid() {
222-
return generateUuid;
223-
}
224-
225215
public String getAddWhen() { return addWhen; }
226216

227-
@AssertTrue(message = "Exactly one of value, format, value_expression, or generate_uuid must be specified")
217+
@AssertTrue(message = "Exactly one of value, format, or value_expression must be specified")
228218
public boolean hasValueOrFormatOrExpression() {
229-
final long count = Stream.of(value, format, valueExpression).filter(n -> n != null).count();
230-
if (generateUuid) {
231-
return count == 0;
232-
}
233-
return count == 1;
219+
return Stream.of(value, format, valueExpression).filter(n -> n!=null).count() == 1;
234220
}
235221

236222
@AssertTrue(message = "overwrite_if_key_exists and append_if_key_exists can not be set to true at the same time.")
@@ -243,46 +229,6 @@ boolean flattenKeyFalseIsUsedWithIterateOn() {
243229
return (!flattenKey && iterateOn!=null) || flattenKey;
244230
}
245231

246-
public Entry(final String key,
247-
final String metadataKey,
248-
final Object value,
249-
final String format,
250-
final String valueExpression,
251-
final boolean overwriteIfKeyExists,
252-
final boolean appendIfKeyExists,
253-
final String addWhen,
254-
final String iterateOn,
255-
final boolean flattenKey,
256-
final String addToElementWhen,
257-
final boolean generateUuid)
258-
{
259-
if (key != null && metadataKey != null) {
260-
throw new IllegalArgumentException("Only one of the two - key and metadatakey - should be specified");
261-
}
262-
if (key == null && metadataKey == null) {
263-
throw new IllegalArgumentException("At least one of the two - key and metadatakey - must be specified");
264-
}
265-
if (metadataKey != null && iterateOn != null) {
266-
throw new IllegalArgumentException("iterate_on cannot be applied to metadata");
267-
}
268-
if (iterateOn == null && addToElementWhen != null) {
269-
throw new InvalidPluginConfigurationException("add_to_element_when only applies when iterate_on is configured.");
270-
}
271-
272-
this.key = key;
273-
this.metadataKey = metadataKey;
274-
this.value = value;
275-
this.format = format;
276-
this.valueExpression = valueExpression;
277-
this.overwriteIfKeyExists = overwriteIfKeyExists;
278-
this.appendIfKeyExists = appendIfKeyExists;
279-
this.addWhen = addWhen;
280-
this.iterateOn = iterateOn;
281-
this.flattenKey = flattenKey;
282-
this.addToElementWhen = addToElementWhen;
283-
this.generateUuid = generateUuid;
284-
}
285-
286232
public Entry(final String key,
287233
final String metadataKey,
288234
final Object value,
@@ -304,7 +250,6 @@ public Entry(final String key,
304250
if (metadataKey != null && iterateOn != null) {
305251
throw new IllegalArgumentException("iterate_on cannot be applied to metadata");
306252
}
307-
308253
if (iterateOn == null && addToElementWhen != null) {
309254
throw new InvalidPluginConfigurationException("add_to_element_when only applies when iterate_on is configured.");
310255
}

‎data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessorTests.java‎

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1089,39 +1089,50 @@ public void testAddFlattenedNestedEntryIterateOn() {
10891089
}
10901090

10911091
@Test
1092-
void test_generate_uuid_adds_uuid_string_to_event() {
1092+
void test_generate_uuid_expression_adds_uuid_string_to_event() {
1093+
final String uuidExpr = "generate_uuid()";
1094+
final String generatedUuid = UUID.randomUUID().toString();
10931095
when(mockConfig.getEntries()).thenReturn(createListOfEntries(
1094-
createEntryWithGenerateUuid("recordId", false, null)));
1096+
createEntry("recordId", null, null, null, uuidExpr, false, false, null, null, null)));
1097+
when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true);
1098+
when(expressionEvaluator.evaluate(eq(uuidExpr), any())).thenReturn(generatedUuid);
10951099

10961100
final AddEntryProcessor processor = createObjectUnderTest();
10971101
final Record<Event> record = getEvent("test-message");
10981102
final List<Record<Event>> result = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
10991103

11001104
final Event event = result.get(0).getData();
11011105
assertThat(event.containsKey("recordId"), is(true));
1102-
final String uuid = event.get("recordId", String.class);
1103-
assertThat(uuid, equalTo(UUID.fromString(uuid).toString()));
1106+
assertThat(event.get("recordId", String.class), equalTo(generatedUuid));
11041107
}
11051108

11061109
@Test
1107-
void test_generate_uuid_produces_unique_values_per_event() {
1110+
void test_generate_uuid_expression_produces_unique_values_per_event() {
1111+
final String uuidExpr = "generate_uuid()";
1112+
final String uuid1 = UUID.randomUUID().toString();
1113+
final String uuid2 = UUID.randomUUID().toString();
11081114
when(mockConfig.getEntries()).thenReturn(createListOfEntries(
1109-
createEntryWithGenerateUuid("recordId", false, null)));
1115+
createEntry("recordId", null, null, null, uuidExpr, false, false, null, null, null)));
1116+
when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true);
1117+
when(expressionEvaluator.evaluate(eq(uuidExpr), any()))
1118+
.thenReturn(uuid1)
1119+
.thenReturn(uuid2);
11101120

11111121
final AddEntryProcessor processor = createObjectUnderTest();
1112-
final Record<Event> record1 = getEvent("message-one");
1113-
final Record<Event> record2 = getEvent("message-two");
1114-
final List<Record<Event>> result = (List<Record<Event>>) processor.doExecute(Arrays.asList(record1, record2));
1122+
final List<Record<Event>> result = (List<Record<Event>>) processor.doExecute(
1123+
Arrays.asList(getEvent("message-one"), getEvent("message-two")));
11151124

1116-
final String uuid1 = result.get(0).getData().get("recordId", String.class);
1117-
final String uuid2 = result.get(1).getData().get("recordId", String.class);
1125+
assertThat(result.get(0).getData().get("recordId", String.class), equalTo(uuid1));
1126+
assertThat(result.get(1).getData().get("recordId", String.class), equalTo(uuid2));
11181127
assertThat(uuid1.equals(uuid2), is(false));
11191128
}
11201129

11211130
@Test
1122-
void test_generate_uuid_does_not_overwrite_existing_key_by_default() {
1131+
void test_generate_uuid_expression_does_not_overwrite_existing_key_by_default() {
1132+
final String uuidExpr = "generate_uuid()";
11231133
when(mockConfig.getEntries()).thenReturn(createListOfEntries(
1124-
createEntryWithGenerateUuid("existingId", false, null)));
1134+
createEntry("existingId", null, null, null, uuidExpr, false, false, null, null, null)));
1135+
when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true);
11251136

11261137
final AddEntryProcessor processor = createObjectUnderTest();
11271138
final Map<String, Object> data = new HashMap<>();
@@ -1133,26 +1144,30 @@ void test_generate_uuid_does_not_overwrite_existing_key_by_default() {
11331144
}
11341145

11351146
@Test
1136-
void test_generate_uuid_overwrites_existing_key_when_overwrite_is_true() {
1147+
void test_generate_uuid_expression_overwrites_existing_key_when_overwrite_is_true() {
1148+
final String uuidExpr = "generate_uuid()";
1149+
final String newUuid = UUID.randomUUID().toString();
11371150
when(mockConfig.getEntries()).thenReturn(createListOfEntries(
1138-
createEntryWithGenerateUuid("existingId", true, null)));
1151+
createEntry("existingId", null, null, null, uuidExpr, true, false, null, null, null)));
1152+
when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true);
1153+
when(expressionEvaluator.evaluate(eq(uuidExpr), any())).thenReturn(newUuid);
11391154

11401155
final AddEntryProcessor processor = createObjectUnderTest();
11411156
final Map<String, Object> data = new HashMap<>();
11421157
data.put("existingId", "original-value");
11431158
final Record<Event> record = buildRecordWithEvent(data);
11441159
final List<Record<Event>> result = (List<Record<Event>>) processor.doExecute(Collections.singletonList(record));
11451160

1146-
final String newValue = result.get(0).getData().get("existingId", String.class);
1147-
assertThat(newValue.equals("original-value"), is(false));
1148-
assertThat(newValue, equalTo(UUID.fromString(newValue).toString()));
1161+
assertThat(result.get(0).getData().get("existingId", String.class), equalTo(newUuid));
11491162
}
11501163

11511164
@Test
1152-
void test_generate_uuid_respects_add_when_condition() {
1165+
void test_generate_uuid_expression_respects_add_when_condition() {
1166+
final String uuidExpr = "generate_uuid()";
11531167
final String addWhen = "/skip == true";
11541168
when(mockConfig.getEntries()).thenReturn(createListOfEntries(
1155-
createEntryWithGenerateUuid("recordId", false, addWhen)));
1169+
createEntry("recordId", null, null, null, uuidExpr, false, false, addWhen, null, null)));
1170+
when(expressionEvaluator.isValidExpressionStatement(uuidExpr)).thenReturn(true);
11561171
when(expressionEvaluator.isValidExpressionStatement(addWhen)).thenReturn(true);
11571172
when(expressionEvaluator.evaluateConditional(eq(addWhen), any())).thenReturn(false);
11581173

@@ -1199,13 +1214,6 @@ private AddEntryProcessorConfig.Entry createEntry(
11991214
iterateOn, addToElementWhen);
12001215
}
12011216

1202-
private AddEntryProcessorConfig.Entry createEntryWithGenerateUuid(
1203-
final String key,
1204-
final boolean overwriteIfKeyExists,
1205-
final String addWhen) {
1206-
return new AddEntryProcessorConfig.Entry(
1207-
key, null, null, null, null, overwriteIfKeyExists, false, addWhen, null, true, null, true);
1208-
}
12091217

12101218
private List<AddEntryProcessorConfig.Entry> createListOfEntries(final AddEntryProcessorConfig.Entry... entries) {
12111219
return new LinkedList<>(Arrays.asList(entries));

0 commit comments

Comments
 (0)