Skip to content

Commit 18a5f23

Browse files
graytaylor0graytaylor0
committed
Add obfuscate_when parameter to obfuscate processor
Signed-off-by: Taylor Gray <tylgry@amazon.com>
1 parent 910f451 commit 18a5f23

3 files changed

Lines changed: 69 additions & 15 deletions

File tree

‎data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.dataprepper.plugins.processor.obfuscation;
77

8+
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
89
import org.opensearch.dataprepper.metrics.PluginMetrics;
910
import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
1011
import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor;
@@ -25,6 +26,7 @@
2526
import java.util.ArrayList;
2627
import java.util.Collection;
2728
import java.util.List;
29+
import java.util.Objects;
2830
import java.util.regex.Matcher;
2931
import java.util.regex.Pattern;
3032

@@ -33,6 +35,10 @@ public class ObfuscationProcessor extends AbstractProcessor<Record<Event>, Recor
3335

3436
private static final String COMMON_PATTERN_REGEX = "^%\\{([A-Z_0-9]+)}$";
3537
private static final Logger LOG = LoggerFactory.getLogger(ObfuscationProcessor.class);
38+
39+
private final ExpressionEvaluator expressionEvaluator;
40+
private final ObfuscationProcessorConfig obfuscationProcessorConfig;
41+
3642
private final String source;
3743
private final String target;
3844

@@ -41,13 +47,20 @@ public class ObfuscationProcessor extends AbstractProcessor<Record<Event>, Recor
4147

4248

4349
@DataPrepperPluginConstructor
44-
public ObfuscationProcessor(final PluginMetrics pluginMetrics, final ObfuscationProcessorConfig config, final PluginFactory pluginFactory) {
50+
public ObfuscationProcessor(final PluginMetrics pluginMetrics,
51+
final ObfuscationProcessorConfig config,
52+
final PluginFactory pluginFactory,
53+
final ExpressionEvaluator expressionEvaluator) {
4554
// No special metrics generate by this processor.
4655
super(pluginMetrics);
4756

4857
this.source = config.getSource();
4958
this.target = config.getTarget();
5059
this.patterns = new ArrayList<>();
60+
this.expressionEvaluator = expressionEvaluator;
61+
this.obfuscationProcessorConfig = config;
62+
63+
config.validateObfuscateWhen(expressionEvaluator);
5164

5265
final PluginModel actionPlugin = config.getAction();
5366
if (actionPlugin == null) {
@@ -94,6 +107,10 @@ public Collection<Record<Event>> doExecute(Collection<Record<Event>> records) {
94107
for (final Record<Event> record : records) {
95108
final Event recordEvent = record.getData();
96109

110+
if (Objects.nonNull(obfuscationProcessorConfig.getObfuscateWhen()) && !expressionEvaluator.evaluateConditional(obfuscationProcessorConfig.getObfuscateWhen(), recordEvent)) {
111+
continue;
112+
}
113+
97114
if (!recordEvent.containsKey(source)) {
98115
continue;
99116
}

‎data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
import com.fasterxml.jackson.annotation.JsonProperty;
99
import jakarta.validation.constraints.NotEmpty;
1010
import jakarta.validation.constraints.NotNull;
11+
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
1112
import org.opensearch.dataprepper.model.configuration.PluginModel;
13+
import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException;
1214

1315
import java.util.List;
1416

@@ -28,6 +30,10 @@ public class ObfuscationProcessorConfig {
2830
@JsonProperty("action")
2931
private PluginModel action;
3032

33+
@JsonProperty("obfuscate_when")
34+
@NotEmpty
35+
private String obfuscateWhen;
36+
3137
public ObfuscationProcessorConfig() {
3238
}
3339

@@ -53,4 +59,14 @@ public String getTarget() {
5359
public PluginModel getAction() {
5460
return action;
5561
}
62+
63+
public String getObfuscateWhen() {
64+
return obfuscateWhen;
65+
}
66+
67+
void validateObfuscateWhen(final ExpressionEvaluator expressionEvaluator) {
68+
if (obfuscateWhen != null && !expressionEvaluator.isValidExpressionStatement(obfuscateWhen)) {
69+
throw new InvalidPluginConfigurationException(String.format("obfuscate_when value %s is not a valid Data Prepper expression statement", obfuscateWhen));
70+
}
71+
}
5672
}

‎data-prepper-plugins/obfuscate-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorTest.java‎

Lines changed: 35 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.junit.jupiter.params.provider.ValueSource;
1414
import org.mockito.Mock;
1515
import org.mockito.junit.jupiter.MockitoExtension;
16+
import org.opensearch.dataprepper.expression.ExpressionEvaluator;
1617
import org.opensearch.dataprepper.metrics.PluginMetrics;
1718
import org.opensearch.dataprepper.model.configuration.PluginModel;
1819
import org.opensearch.dataprepper.model.configuration.PluginSetting;
@@ -27,6 +28,7 @@
2728
import java.util.HashMap;
2829
import java.util.List;
2930
import java.util.Map;
31+
import java.util.UUID;
3032

3133
import static org.hamcrest.MatcherAssert.assertThat;
3234
import static org.hamcrest.Matchers.equalTo;
@@ -53,6 +55,9 @@ class ObfuscationProcessorTest {
5355
@Mock
5456
private ObfuscationProcessorConfig mockConfig;
5557

58+
@Mock
59+
private ExpressionEvaluator expressionEvaluator;
60+
5661
private ObfuscationProcessor obfuscationProcessor;
5762

5863
static Record<Event> buildRecordWithEvent(final Map<String, Object> data) {
@@ -75,7 +80,23 @@ void setup() {
7580
lenient().when(mockConfig.getAction()).thenReturn(defaultConfig.getAction());
7681
lenient().when(mockConfig.getPatterns()).thenReturn(defaultConfig.getPatterns());
7782
lenient().when(mockConfig.getTarget()).thenReturn(defaultConfig.getTarget());
78-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
83+
lenient().when(mockConfig.getObfuscateWhen()).thenReturn(null);
84+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
85+
}
86+
87+
@Test
88+
void obfuscate_when_evaluates_to_false_does_not_modify_event() {
89+
final String expression = "/test == success";
90+
final Record<Event> record = createRecord(UUID.randomUUID().toString());
91+
when(mockConfig.getObfuscateWhen()).thenReturn(expression);
92+
when(expressionEvaluator.evaluateConditional(expression, record.getData())).thenReturn(false);
93+
94+
final ObfuscationProcessor objectUnderTest = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
95+
96+
final List<Record<Event>> editedRecords = (List<Record<Event>>) objectUnderTest.doExecute(Collections.singletonList(record));
97+
98+
assertThat(editedRecords.size(), equalTo(1));
99+
assertThat(editedRecords.get(0), equalTo(record));
79100
}
80101

81102

@@ -102,7 +123,7 @@ void testProcessorWithDifferentAction() {
102123

103124
when(mockFactory.loadPlugin(eq(ObfuscationAction.class), any(PluginSetting.class)))
104125
.thenReturn(mockAction);
105-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
126+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
106127

107128
final Record<Event> record = createRecord("Hello");
108129
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -117,7 +138,7 @@ void testProcessorWithDifferentAction() {
117138
@ValueSource(strings = {"hello", "hello, world", "This is a message", "123", "你好"})
118139
void testProcessorWithTarget(String message) {
119140
when(mockConfig.getTarget()).thenReturn("new_message");
120-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
141+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
121142

122143
final Record<Event> record = createRecord(message);
123144
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -135,7 +156,7 @@ void testProcessorWithTarget(String message) {
135156
@Test
136157
void testProcessorWithUnknownSource() {
137158
when(mockConfig.getSource()).thenReturn("email");
138-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
159+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
139160

140161
final Record<Event> record = createRecord("Hello");
141162
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -158,7 +179,7 @@ void testProcessorWithUnknownSource() {
158179
})
159180
void testProcessorWithPattern(String message, String pattern, String expected) {
160181
when(mockConfig.getPatterns()).thenReturn(List.of(pattern));
161-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
182+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
162183

163184
final Record<Event> record = createRecord(message);
164185
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -171,13 +192,13 @@ void testProcessorWithPattern(String message, String pattern, String expected) {
171192
@Test
172193
void testProcessorWithUnknownPattern() {
173194
when(mockConfig.getPatterns()).thenReturn(List.of("%{UNKNOWN}"));
174-
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory));
195+
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator));
175196
}
176197

177198
@Test
178199
void testProcessorInvalidPattern() {
179200
when(mockConfig.getPatterns()).thenReturn(List.of("["));
180-
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory));
201+
assertThrows(InvalidPluginConfigurationException.class, () -> new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator));
181202
}
182203

183204
@ParameterizedTest
@@ -194,7 +215,7 @@ void testProcessorInvalidPattern() {
194215
})
195216
void testProcessorWithEmailAddressPattern(String message, String expected) {
196217
when(mockConfig.getPatterns()).thenReturn(List.of("%{EMAIL_ADDRESS}"));
197-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
218+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
198219

199220
final Record<Event> record = createRecord(message);
200221

@@ -221,7 +242,7 @@ void testProcessorWithEmailAddressPattern(String message, String expected) {
221242
})
222243
void testProcessorWithUSPhoneNumberPattern(String message, String expected) {
223244
when(mockConfig.getPatterns()).thenReturn(List.of("%{US_PHONE_NUMBER}"));
224-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
245+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
225246

226247
final Record<Event> record = createRecord(message);
227248

@@ -247,7 +268,7 @@ void testProcessorWithUSPhoneNumberPattern(String message, String expected) {
247268
})
248269
void testProcessorWithCreditNumberPattern(String message, String expected) {
249270
when(mockConfig.getPatterns()).thenReturn(List.of("%{CREDIT_CARD_NUMBER}"));
250-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
271+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
251272

252273
final Record<Event> record = createRecord(message);
253274

@@ -271,7 +292,7 @@ void testProcessorWithCreditNumberPattern(String message, String expected) {
271292
})
272293
void testProcessorWithIPAddressV4Pattern(String message, String expected) {
273294
when(mockConfig.getPatterns()).thenReturn(List.of("%{IP_ADDRESS_V4}"));
274-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
295+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
275296

276297
final Record<Event> record = createRecord(message);
277298
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -291,7 +312,7 @@ void testProcessorWithIPAddressV4Pattern(String message, String expected) {
291312
})
292313
void testProcessorWithUSSSNPattern(String message, String expected) {
293314
when(mockConfig.getPatterns()).thenReturn(List.of("%{US_SSN_NUMBER}"));
294-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
315+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
295316

296317
final Record<Event> record = createRecord(message);
297318
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -314,7 +335,7 @@ void testProcessorWithUSSSNPattern(String message, String expected) {
314335
})
315336
void testProcessorWithBaseNumberPattern(String message, String expected) {
316337
when(mockConfig.getPatterns()).thenReturn(List.of("%{BASE_NUMBER}"));
317-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
338+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
318339

319340
final Record<Event> record = createRecord(message);
320341
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));
@@ -333,7 +354,7 @@ void testProcessorWithBaseNumberPattern(String message, String expected) {
333354
})
334355
void testProcessorWithMultiplePatterns(String message, String expected) {
335356
when(mockConfig.getPatterns()).thenReturn(List.of("%{EMAIL_ADDRESS}", "%{IP_ADDRESS_V4}"));
336-
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory);
357+
obfuscationProcessor = new ObfuscationProcessor(pluginMetrics, mockConfig, mockFactory, expressionEvaluator);
337358

338359
final Record<Event> record = createRecord(message);
339360
final List<Record<Event>> editedRecords = (List<Record<Event>>) obfuscationProcessor.doExecute(Collections.singletonList(record));

0 commit comments

Comments
 (0)