Skip to content

Commit 0a48278

Browse files
kennedy-onyiakennedy-onyia
authored
Add regex-based key selection for delete_entries processor (opensearch-project#6146)
Add regex-based key selection for delete_entries processor Signed-off-by: Kennedy Onyia <kennedy.onyia@gmail.com>
1 parent fb04160 commit 0a48278

4 files changed

Lines changed: 471 additions & 35 deletions

File tree

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java‎

Lines changed: 48 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,22 @@
2222
import org.slf4j.LoggerFactory;
2323

2424
import java.util.Collection;
25+
import java.util.Collections;
2526
import java.util.List;
2627
import java.util.Map;
2728
import java.util.Objects;
29+
import java.util.Set;
30+
import java.util.regex.Pattern;
31+
import java.util.stream.Collectors;
2832

2933
@DataPrepperPlugin(name = "delete_entries", pluginType = Processor.class, pluginConfigurationType = DeleteEntryProcessorConfig.class)
3034
public class DeleteEntryProcessor extends AbstractProcessor<Record<Event>, Record<Event>> {
3135

3236
private static final Logger LOG = LoggerFactory.getLogger(DeleteEntryProcessor.class);
3337
private final List<EventKey> withKeys;
38+
private final List<String> withKeysRegex;
39+
private final List<Pattern> withKeysRegexPattern;
40+
private final Set<EventKey> excludeFromDelete;
3441
private final String deleteWhen;
3542
private final List<DeleteEntryProcessorConfig.Entry> entries;
3643

@@ -41,6 +48,9 @@ public class DeleteEntryProcessor extends AbstractProcessor<Record<Event>, Recor
4148
public DeleteEntryProcessor(final PluginMetrics pluginMetrics, final DeleteEntryProcessorConfig config, final ExpressionEvaluator expressionEvaluator) {
4249
super(pluginMetrics);
4350
this.withKeys = config.getWithKeys();
51+
this.withKeysRegex = config.getWithKeysRegex();
52+
this.withKeysRegexPattern = config.getWithKeysRegexPattern();
53+
this.excludeFromDelete = config.getExcludeFromDelete();
4454
this.deleteEntryProcessorConfig = config;
4555
this.deleteWhen = config.getDeleteWhen();
4656
this.expressionEvaluator = expressionEvaluator;
@@ -52,8 +62,14 @@ public DeleteEntryProcessor(final PluginMetrics pluginMetrics, final DeleteEntry
5262
".org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", deleteWhen));
5363
}
5464

55-
if (this.withKeys != null && !this.withKeys.isEmpty()) {
56-
DeleteEntryProcessorConfig.Entry entry = new DeleteEntryProcessorConfig.Entry(this.withKeys, this.deleteWhen, config.getIterateOn(), config.getDeleteFromElementWhen());
65+
if (!this.withKeys.isEmpty() || !this.withKeysRegex.isEmpty()) {
66+
DeleteEntryProcessorConfig.Entry entry = new DeleteEntryProcessorConfig.Entry(
67+
this.withKeys,
68+
this.withKeysRegex,
69+
this.excludeFromDelete,
70+
this.deleteWhen,
71+
config.getIterateOn(),
72+
config.getDeleteFromElementWhen());
5773
this.entries = List.of(entry);
5874
} else {
5975
this.entries = config.getEntries();
@@ -92,12 +108,9 @@ public Collection<Record<Event>> doExecute(final Collection<Record<Event>> recor
92108
continue;
93109
}
94110

95-
final String iterateOn = deleteEntryProcessorConfig.getIterateOn();
111+
final String iterateOn = entry.getIterateOn();
96112
if (Objects.isNull(iterateOn)) {
97-
98-
for (final EventKey entryKey : entry.getWithKeys()) {
99-
recordEvent.delete(entryKey);
100-
}
113+
deleteKeysFromEvent(recordEvent, entry);
101114
} else {
102115
handleForIterateOn(recordEvent, entry, iterateOn);
103116
}
@@ -128,6 +141,33 @@ public boolean isReadyForShutdown() {
128141
public void shutdown() {
129142
}
130143

144+
private void deleteKeysFromEvent(final Event event, final DeleteEntryProcessorConfig.Entry entry) {
145+
if (!entry.getWithKeys().isEmpty()) {
146+
for (final EventKey entryKey : entry.getWithKeys()) {
147+
event.delete(entryKey);
148+
}
149+
}
150+
151+
if (!entry.getWithKeysRegex().isEmpty()) {
152+
final Set<String> excludeKeys = entry.getExcludeFromDelete() == null ? Collections.emptySet()
153+
: entry.getExcludeFromDelete().stream()
154+
.map(EventKey::getKey)
155+
.collect(Collectors.toSet());
156+
157+
final List<Pattern> validRegexPatterns = entry.getWithKeysRegexPattern();
158+
159+
for (final String key : event.toMap().keySet()) {
160+
for (final Pattern regexPattern : validRegexPatterns) {
161+
if (!excludeKeys.contains(key)) {
162+
if (regexPattern.matcher(key).matches()) {
163+
event.delete(key);
164+
}
165+
}
166+
}
167+
}
168+
}
169+
}
170+
131171
private void handleForIterateOn(final Event recordEvent,
132172
final DeleteEntryProcessorConfig.Entry entry,
133173
final String iterateOn) {
@@ -144,9 +184,7 @@ private void handleForIterateOn(final Event recordEvent,
144184
continue;
145185
}
146186

147-
for (final EventKey entryKey : entry.getWithKeys()) {
148-
context.delete(entryKey);
149-
}
187+
deleteKeysFromEvent(context, entry);
150188
iterateOnList.set(i, context.toMap());
151189
}
152190
recordEvent.put(iterateOn, iterateOnList);

‎data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfig.java‎

Lines changed: 129 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
package org.opensearch.dataprepper.plugins.processor.mutateevent;
77

88
import com.fasterxml.jackson.annotation.JsonClassDescription;
9+
import com.fasterxml.jackson.annotation.JsonIgnore;
910
import com.fasterxml.jackson.annotation.JsonProperty;
1011
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
1112
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
1213
import jakarta.validation.Valid;
13-
import jakarta.validation.constraints.AssertFalse;
1414
import jakarta.validation.constraints.AssertTrue;
1515
import jakarta.validation.constraints.NotEmpty;
1616
import jakarta.validation.constraints.NotNull;
@@ -23,33 +23,49 @@
2323
import org.opensearch.dataprepper.model.event.EventKeyConfiguration;
2424
import org.opensearch.dataprepper.model.event.EventKeyFactory;
2525

26+
import java.util.Collections;
2627
import java.util.List;
28+
import java.util.Set;
29+
import java.util.regex.Pattern;
30+
import java.util.regex.PatternSyntaxException;
31+
import java.util.stream.Collectors;
2732

2833
@ConditionalRequired(value = {
2934
@IfThenElse(
30-
ifFulfilled = {@SchemaProperty(field = "entries", value = "null")},
35+
ifFulfilled = {@SchemaProperty(field = "entries", value = "null"), @SchemaProperty(field = "with_keys_regex", value = "null")},
3136
thenExpect = {@SchemaProperty(field = "with_keys")}
3237
),
3338
@IfThenElse(
34-
ifFulfilled = {@SchemaProperty(field = "with_keys", value = "null")},
39+
ifFulfilled = {@SchemaProperty(field = "entries", value = "null"), @SchemaProperty(field = "with_keys", value = "null")},
40+
thenExpect = {@SchemaProperty(field = "with_keys_regex")}
41+
),
42+
@IfThenElse(
43+
ifFulfilled = {@SchemaProperty(field = "with_keys", value = "null"), @SchemaProperty(field = "with_keys_regex", value = "null")},
3544
thenExpect = {@SchemaProperty(field = "entries")}
3645
)
3746
})
3847
@JsonPropertyOrder
3948
@JsonClassDescription("The <code>delete_entries</code> processor deletes fields from events. " +
40-
"You can define the keys you want to delete in the <code>with_keys</code> configuration. " +
41-
"Those keys and their values are deleted from events.")
49+
"You can specify the keys of the fields you want to delete using the <code>with_keys</code> " +
50+
"or <code>with_keys_regex</code> configuration options. You can only use one per entry. " +
51+
"Those keys and their values are deleted from the events.")
4252
public class DeleteEntryProcessorConfig {
4353

4454
@JsonPropertyOrder
4555
public static class Entry {
46-
@NotEmpty
47-
@NotNull
4856
@JsonProperty("with_keys")
4957
@EventKeyConfiguration(EventKeyFactory.EventAction.DELETE)
5058
@JsonPropertyDescription("A list of keys to be deleted.")
5159
private List<@NotNull @NotEmpty EventKey> withKeys;
5260

61+
@JsonProperty("with_keys_regex")
62+
@JsonPropertyDescription("A list of regex patterns to match keys to be deleted.")
63+
private List<String> withKeysRegex;
64+
65+
@JsonProperty("exclude_from_delete")
66+
@JsonPropertyDescription("A list of keys to exclude from deletion when using with_keys_regex.")
67+
private Set<EventKey> excludeFromDelete;
68+
5369
@JsonProperty("delete_when")
5470
@JsonPropertyDescription("Specifies under what condition the deletion should be performed. " +
5571
"By default, keys are always deleted. Example: <code>/mykey == \"---\"</code>")
@@ -67,8 +83,49 @@ public static class Entry {
6783
@JsonProperty("iterate_on")
6884
private String iterateOn;
6985

86+
@AssertTrue(message = "exclude_from_delete only applies when with_keys_regex is configured.")
87+
boolean isExcludeFromDeleteValid() {
88+
return excludeFromDelete != null && !excludeFromDelete.isEmpty() && withKeysRegex != null && !withKeysRegex.isEmpty();
89+
}
90+
91+
@JsonIgnore
92+
private List<Pattern> withKeysRegexPatterns;
93+
94+
private void setWithKeysRegexPatterns() {
95+
withKeysRegexPatterns = getWithKeysRegex().stream()
96+
.map(Pattern::compile)
97+
.collect(Collectors.toList());
98+
}
99+
100+
@AssertTrue(message = "Invalid regex pattern found in with_keys_regex.")
101+
public boolean isValidWithKeysRegexPattern() {
102+
if (withKeysRegex != null && !withKeysRegex.isEmpty()) {
103+
try {
104+
setWithKeysRegexPatterns();
105+
} catch (PatternSyntaxException e) {
106+
return false;
107+
}
108+
}
109+
return true;
110+
}
111+
70112
public List<EventKey> getWithKeys() {
71-
return withKeys;
113+
return withKeys != null ? withKeys : Collections.emptyList();
114+
}
115+
116+
public List<String> getWithKeysRegex() {
117+
return withKeysRegex != null ? withKeysRegex : Collections.emptyList();
118+
}
119+
120+
public List<Pattern> getWithKeysRegexPattern() {
121+
if (withKeysRegexPatterns == null && withKeysRegex != null && !withKeysRegex.isEmpty()) {
122+
setWithKeysRegexPatterns();
123+
}
124+
return withKeysRegexPatterns != null ? withKeysRegexPatterns : Collections.emptyList();
125+
}
126+
127+
public Set<EventKey> getExcludeFromDelete() {
128+
return excludeFromDelete != null ? excludeFromDelete : Collections.emptySet();
72129
}
73130

74131
public String getDeleteWhen() {
@@ -82,17 +139,20 @@ public String getDeleteFromElementWhen() {
82139
}
83140

84141
public Entry(final List<EventKey> withKeys,
142+
final List<String> withKeysRegex,
143+
final Set<EventKey> excludeFromDelete,
85144
final String deleteWhen,
86145
final String iterateOn,
87146
final String deleteFromElementWhen) {
88147
this.withKeys = withKeys;
148+
this.withKeysRegex = withKeysRegex;
149+
this.excludeFromDelete = excludeFromDelete;
89150
this.deleteWhen = deleteWhen;
90151
this.deleteFromElementWhen = deleteFromElementWhen;
91152
this.iterateOn = iterateOn;
92153
}
93154

94155
public Entry() {
95-
96156
}
97157
}
98158

@@ -101,6 +161,14 @@ public Entry() {
101161
@JsonPropertyDescription("A list of keys to be deleted. May not be used with entries.")
102162
private List<@NotNull @NotEmpty EventKey> withKeys;
103163

164+
@JsonProperty("with_keys_regex")
165+
@JsonPropertyDescription("A list of regex patterns that match keys to be deleted from an event. May not be used with entries.")
166+
private List<String> withKeysRegex;
167+
168+
@JsonProperty("exclude_from_delete")
169+
@JsonPropertyDescription("A list of keys to exclude from deletion when using with_keys_regex.")
170+
private Set<EventKey> excludeFromDelete;
171+
104172
@JsonProperty("delete_when")
105173
@JsonPropertyDescription("Specifies under what condition the deletion should be performed.")
106174
private String deleteWhen;
@@ -110,15 +178,46 @@ public Entry() {
110178
@JsonPropertyDescription("A list of entries to delete from the event.")
111179
private List<Entry> entries;
112180

113-
@AssertTrue(message = "Either 'entries' or 'with_keys' must be specified, but neither was found")
181+
@AssertTrue(message = "One of the following must be provided: 'entries', 'with_keys', or 'with_keys_regex'. None of these are configured.")
114182
boolean isConfigurationPresent() {
115-
return entries != null || withKeys != null;
183+
return entries != null || withKeys != null || withKeysRegex != null;
184+
}
185+
186+
@AssertTrue(message = "You can only use one of the following at a time: 'entries', 'with_keys', or 'with_keys_regex'")
187+
boolean hasOnlyOneConfiguration() {
188+
int count = 0;
189+
if (entries != null) count++;
190+
if (withKeys != null) count++;
191+
if (withKeysRegex != null) count++;
192+
return count == 1;
193+
}
194+
195+
@AssertTrue(message = "exclude_from_delete only applies when with_keys_regex is configured.")
196+
boolean isExcludeFromDeleteValid() {
197+
return excludeFromDelete != null && !excludeFromDelete.isEmpty() && withKeysRegex != null && !withKeysRegex.isEmpty();
116198
}
117199

118-
@AssertFalse(message = "Either use 'entries' OR 'with_keys' with 'delete_when' configuration, but not both")
119-
boolean hasBothConfigurations() {
120-
return entries != null && withKeys != null;
200+
@JsonIgnore
201+
private List<Pattern> withKeysRegexPatterns;
202+
203+
private void setWithKeysRegexPatterns() {
204+
withKeysRegexPatterns = getWithKeysRegex().stream()
205+
.map(Pattern::compile)
206+
.collect(Collectors.toList());
207+
}
208+
209+
@AssertTrue(message = "Invalid regex pattern in 'with_keys_regex'")
210+
boolean isValidWithKeysRegexPattern() {
211+
if (withKeysRegex != null && !withKeysRegex.isEmpty()) {
212+
try {
213+
setWithKeysRegexPatterns();
214+
} catch (PatternSyntaxException e) {
215+
return false;
216+
}
217+
}
218+
return true;
121219
}
220+
122221
@JsonPropertyDescription(
123222
"Specifies the key of the list of object to iterate over and delete the keys specified in with_keys.")
124223
@JsonProperty("iterate_on")
@@ -129,7 +228,22 @@ boolean hasBothConfigurations() {
129228
private String deleteFromElementWhen;
130229

131230
public List<EventKey> getWithKeys() {
132-
return withKeys;
231+
return withKeys != null ? withKeys : Collections.emptyList();
232+
}
233+
234+
public List<String> getWithKeysRegex() {
235+
return withKeysRegex != null ? withKeysRegex : Collections.emptyList();
236+
}
237+
238+
public List<Pattern> getWithKeysRegexPattern() {
239+
if (withKeysRegexPatterns == null && withKeysRegex != null && !withKeysRegex.isEmpty()) {
240+
setWithKeysRegexPatterns();
241+
}
242+
return withKeysRegexPatterns != null ? withKeysRegexPatterns : Collections.emptyList();
243+
}
244+
245+
public Set<EventKey> getExcludeFromDelete() {
246+
return excludeFromDelete != null ? excludeFromDelete : Collections.emptySet();
133247
}
134248

135249
public String getDeleteWhen() {

‎data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessorConfigTests.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package org.opensearch.dataprepper.plugins.processor.mutateevent;
2+
3+
import org.junit.jupiter.api.Test;
4+
import org.opensearch.dataprepper.model.event.EventKey;
5+
import org.opensearch.dataprepper.test.helper.ReflectivelySetField;
6+
7+
import java.util.List;
8+
import java.util.Set;
9+
10+
import static org.hamcrest.CoreMatchers.equalTo;
11+
import static org.hamcrest.MatcherAssert.assertThat;
12+
import static org.mockito.Mockito.mock;
13+
14+
public class DeleteEntryProcessorConfigTests {
15+
16+
@Test
17+
void testIsValidKeysRegexPatterns_with_valid_pattern() throws NoSuchFieldException, IllegalAccessException {
18+
final DeleteEntryProcessorConfig objectUnderTest = new DeleteEntryProcessorConfig();
19+
final List<String> validPatterns = List.of("test.*");
20+
ReflectivelySetField.setField(DeleteEntryProcessorConfig.class, objectUnderTest, "withKeysRegex", validPatterns);
21+
22+
assertThat(objectUnderTest.isValidWithKeysRegexPattern(), equalTo(true));
23+
}
24+
25+
@Test
26+
void testIsValidKeysRegexPatterns_with_invalid_pattern() throws NoSuchFieldException, IllegalAccessException {
27+
final DeleteEntryProcessorConfig objectUnderTest = new DeleteEntryProcessorConfig();
28+
final List<String> invalidPatterns = List.of("(abc");
29+
ReflectivelySetField.setField(DeleteEntryProcessorConfig.class, objectUnderTest, "withKeysRegex", invalidPatterns);
30+
31+
assertThat(objectUnderTest.isValidWithKeysRegexPattern(), equalTo(false));
32+
}
33+
34+
@Test
35+
void testisExcludeFromDeleteValid_with_valid_config() throws NoSuchFieldException, IllegalAccessException{
36+
final DeleteEntryProcessorConfig objectUnderTest = new DeleteEntryProcessorConfig();
37+
final List<String> regexKeys = List.of("test.*");
38+
final Set<EventKey> excludeKeys = Set.of(mock(EventKey.class));
39+
ReflectivelySetField.setField(DeleteEntryProcessorConfig.class, objectUnderTest, "withKeysRegex", regexKeys);
40+
ReflectivelySetField.setField(DeleteEntryProcessorConfig.class, objectUnderTest, "excludeFromDelete", excludeKeys);
41+
42+
assertThat(objectUnderTest.isExcludeFromDeleteValid(), equalTo(true));
43+
}
44+
45+
@Test
46+
void testisExcludeFromDeleteValid_with_invalid_config() throws NoSuchFieldException, IllegalAccessException{
47+
final DeleteEntryProcessorConfig objectUnderTest = new DeleteEntryProcessorConfig();
48+
final List<EventKey> testKeys = List.of(mock(EventKey.class));
49+
final Set<EventKey> excludeKeys = Set.of(mock(EventKey.class));
50+
ReflectivelySetField.setField(DeleteEntryProcessorConfig.class, objectUnderTest, "withKeys", testKeys);
51+
ReflectivelySetField.setField(DeleteEntryProcessorConfig.class, objectUnderTest, "excludeFromDelete", excludeKeys);
52+
53+
assertThat(objectUnderTest.isExcludeFromDeleteValid(), equalTo(false));
54+
}
55+
}

0 commit comments

Comments
 (0)