Skip to content

Commit fbca3db

Browse files
kkondakakkondaka
authored
Add TransformFunctionProvider to DynamicConfigTransformer (#6881)
* Add TransformFunctionProvider to DynamicConfigTransformer Signed-off-by: Kondaka <krishkdk@amazon.com> * Fixed checkstyle errors Signed-off-by: Kondaka <krishkdk@amazon.com> * Fixed License headers Signed-off-by: Kondaka <krishkdk@amazon.com> * Fixed failing coverage tests in aws-plugin Signed-off-by: Kondaka <krishkdk@amazon.com> * Addressed review comments Signed-off-by: Kondaka <krishkdk@amazon.com> * Addressed review comments Signed-off-by: Kondaka <krishkdk@amazon.com> * Fixed integration test failures Signed-off-by: Kondaka <krishkdk@amazon.com> * Addressed review comments Signed-off-by: Kondaka <krishkdk@amazon.com> --------- Signed-off-by: Kondaka <krishkdk@amazon.com>
1 parent e54ec6e commit fbca3db

30 files changed

Lines changed: 1355 additions & 151 deletions

File tree

‎data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/TransformationFunction.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.model.annotations;
11+
12+
import java.lang.annotation.Documented;
13+
import java.lang.annotation.ElementType;
14+
import java.lang.annotation.Retention;
15+
import java.lang.annotation.RetentionPolicy;
16+
import java.lang.annotation.Target;
17+
18+
/**
19+
* Marks a method as a pipeline transformation function that can be invoked
20+
* dynamically from template YAML files via the {@code FUNCTION_NAME} placeholder.
21+
* <p>
22+
* Annotated methods must be {@code public static} and accept a single {@code String}
23+
* parameter, returning a {@code String} result.
24+
* <p>
25+
* The enclosing class must implement
26+
* {@link org.opensearch.dataprepper.model.plugin.PipelineTransformFunctionProvider}.
27+
*
28+
* @since 2.12
29+
*/
30+
@Documented
31+
@Retention(RetentionPolicy.RUNTIME)
32+
@Target({ElementType.METHOD})
33+
public @interface TransformationFunction {
34+
}

‎data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PipelineTransformFunctionProvider.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.model.plugin;
11+
12+
/**
13+
* Marker interface for classes that provide pipeline transformation functions.
14+
* Classes implementing this interface can be referenced in rule YAML files via
15+
* the {@code function_providers} field and have their methods invoked dynamically
16+
* during pipeline template transformation.
17+
* <p>
18+
* Methods intended to be callable from templates must also be annotated with
19+
* {@link org.opensearch.dataprepper.model.annotations.TransformationFunction}.
20+
*
21+
* @since 2.12
22+
*/
23+
public interface PipelineTransformFunctionProvider {
24+
}

‎data-prepper-api/src/test/java/org/opensearch/dataprepper/model/annotations/TransformationFunctionTest.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.model.annotations;
11+
12+
import org.junit.jupiter.api.Test;
13+
14+
import java.lang.annotation.Documented;
15+
import java.lang.annotation.ElementType;
16+
import java.lang.annotation.Retention;
17+
import java.lang.annotation.RetentionPolicy;
18+
import java.lang.annotation.Target;
19+
20+
import static org.hamcrest.MatcherAssert.assertThat;
21+
import static org.hamcrest.Matchers.equalTo;
22+
import static org.junit.jupiter.api.Assertions.assertNotNull;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
class TransformationFunctionTest {
26+
27+
@Test
28+
void annotation_is_retained_at_runtime() {
29+
Retention retention = TransformationFunction.class.getAnnotation(Retention.class);
30+
assertNotNull(retention);
31+
assertThat(retention.value(), equalTo(RetentionPolicy.RUNTIME));
32+
}
33+
34+
@Test
35+
void annotation_targets_methods() {
36+
Target target = TransformationFunction.class.getAnnotation(Target.class);
37+
assertNotNull(target);
38+
assertThat(target.value().length, equalTo(1));
39+
assertThat(target.value()[0], equalTo(ElementType.METHOD));
40+
}
41+
42+
@Test
43+
void annotation_is_documented() {
44+
Documented documented = TransformationFunction.class.getAnnotation(Documented.class);
45+
assertNotNull(documented);
46+
}
47+
48+
@Test
49+
void annotation_is_present_on_annotated_method() throws NoSuchMethodException {
50+
assertTrue(AnnotatedClass.class.getMethod("annotatedMethod", String.class)
51+
.isAnnotationPresent(TransformationFunction.class));
52+
}
53+
54+
@Test
55+
void annotation_is_not_present_on_unannotated_method() throws NoSuchMethodException {
56+
assertTrue(!AnnotatedClass.class.getMethod("unannotatedMethod", String.class)
57+
.isAnnotationPresent(TransformationFunction.class));
58+
}
59+
60+
static class AnnotatedClass {
61+
@TransformationFunction
62+
public static String annotatedMethod(String input) {
63+
return input;
64+
}
65+
66+
public static String unannotatedMethod(String input) {
67+
return input;
68+
}
69+
}
70+
}

‎data-prepper-api/src/test/java/org/opensearch/dataprepper/model/plugin/PipelineTransformFunctionProviderTest.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.model.plugin;
11+
12+
import org.junit.jupiter.api.Test;
13+
14+
import static org.hamcrest.MatcherAssert.assertThat;
15+
import static org.hamcrest.Matchers.equalTo;
16+
import static org.junit.jupiter.api.Assertions.assertNotNull;
17+
import static org.junit.jupiter.api.Assertions.assertTrue;
18+
19+
class PipelineTransformFunctionProviderTest {
20+
21+
@Test
22+
void implementing_class_is_assignable_from_interface() {
23+
PipelineTransformFunctionProvider provider = new TestFunctionProvider();
24+
assertNotNull(provider);
25+
assertTrue(provider instanceof PipelineTransformFunctionProvider);
26+
}
27+
28+
@Test
29+
void interface_is_assignable_from_implementing_class() {
30+
assertThat(PipelineTransformFunctionProvider.class.isAssignableFrom(TestFunctionProvider.class),
31+
equalTo(true));
32+
}
33+
34+
@Test
35+
void interface_is_not_assignable_from_non_implementing_class() {
36+
assertThat(PipelineTransformFunctionProvider.class.isAssignableFrom(NonProvider.class),
37+
equalTo(false));
38+
}
39+
40+
@Test
41+
void interface_has_no_declared_methods() {
42+
assertThat(PipelineTransformFunctionProvider.class.getDeclaredMethods().length, equalTo(0));
43+
}
44+
45+
@Test
46+
void interface_is_public() {
47+
assertTrue(java.lang.reflect.Modifier.isPublic(PipelineTransformFunctionProvider.class.getModifiers()));
48+
}
49+
50+
@Test
51+
void interface_is_an_interface() {
52+
assertTrue(PipelineTransformFunctionProvider.class.isInterface());
53+
}
54+
55+
static class TestFunctionProvider implements PipelineTransformFunctionProvider {
56+
public static String sampleFunction(String input) {
57+
return input;
58+
}
59+
}
60+
61+
static class NonProvider {
62+
public static String someMethod(String input) {
63+
return input;
64+
}
65+
}
66+
}

‎data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluator.java‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
48
*/
9+
510
package org.opensearch.dataprepper.pipeline.parser.rule;
611

12+
import org.opensearch.dataprepper.model.annotations.TransformationFunction;
13+
import org.opensearch.dataprepper.model.plugin.PipelineTransformFunctionProvider;
14+
715
import com.fasterxml.jackson.core.JsonProcessingException;
816
import com.fasterxml.jackson.databind.JsonNode;
917
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -22,6 +30,8 @@
2230
import org.slf4j.LoggerFactory;
2331

2432
import java.io.FileNotFoundException;
33+
import java.util.Arrays;
34+
2535
import java.io.IOException;
2636
import java.io.InputStream;
2737
import java.util.ArrayList;
@@ -34,6 +44,7 @@ public class RuleEvaluator {
3444
private static final Logger LOG = LoggerFactory.getLogger(RuleEvaluator.class);
3545
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
3646
private static final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
47+
private static final String REQUIRED_PACKAGE_SEGMENT = "dataprepper_transformer";
3748
private final TransformersFactory transformersFactory;
3849

3950

@@ -62,6 +73,7 @@ public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelin
6273
return RuleEvaluatorResult.builder()
6374
.withEvaluatedResult(true)
6475
.withPipelineTemplateModel(templateModel)
76+
.withFunctionProviders(ruleFileEvaluation.getFunctionProviders())
6577
.withPipelineName(entry.getKey())
6678
.build();
6779
}
@@ -76,6 +88,7 @@ public RuleEvaluatorResult isTransformationNeeded(PipelinesDataFlowModel pipelin
7688
return RuleEvaluatorResult.builder()
7789
.withEvaluatedResult(false)
7890
.withPipelineName(null)
91+
.withFunctionProviders(null)
7992
.withPipelineTemplateModel(null)
8093
.build();
8194
}
@@ -128,9 +141,11 @@ private RuleFileEvaluation evaluate(String pipelinesJson) {
128141
}
129142

130143
if (allRulesValid) {
144+
validateFunctionProviders(rulesModel.getFunctionProviders(), parsedRule.fileName);
131145
return RuleFileEvaluation.builder()
132146
.withPluginName(pluginName)
133147
.withRuleFileName(parsedRule.fileName)
148+
.withFunctionProviders(rulesModel.getFunctionProviders())
134149
.withResult(true)
135150
.build();
136151
}
@@ -154,6 +169,42 @@ private RuleFileEvaluation evaluate(String pipelinesJson) {
154169
.build();
155170
}
156171

172+
private void validateFunctionProviders(List<String> functionProviders, String ruleFileName) {
173+
if (functionProviders == null || functionProviders.isEmpty()) {
174+
return;
175+
}
176+
for (String provider : functionProviders) {
177+
if (!provider.contains(REQUIRED_PACKAGE_SEGMENT)) {
178+
throw new RuntimeException("Invalid function_provider '" + provider +
179+
"' in rule file '" + ruleFileName +
180+
"'. Package must contain '" + REQUIRED_PACKAGE_SEGMENT + "'");
181+
}
182+
183+
final Class<?> clazz;
184+
try {
185+
clazz = Class.forName(provider, false, this.getClass().getClassLoader());
186+
} catch (ClassNotFoundException e) {
187+
throw new RuntimeException("function_provider class '" + provider +
188+
"' in rule file '" + ruleFileName + "' could not be found", e);
189+
}
190+
191+
if (!PipelineTransformFunctionProvider.class.isAssignableFrom(clazz)) {
192+
throw new RuntimeException("function_provider class '" + provider +
193+
"' in rule file '" + ruleFileName +
194+
"' does not implement PipelineTransformFunctionProvider");
195+
}
196+
197+
boolean hasAnnotatedMethod = Arrays.stream(clazz.getMethods())
198+
.anyMatch(m -> m.isAnnotationPresent(TransformationFunction.class));
199+
if (!hasAnnotatedMethod) {
200+
throw new RuntimeException("function_provider class '" + provider +
201+
"' in rule file '" + ruleFileName +
202+
"' has no methods annotated with @TransformationFunction");
203+
}
204+
}
205+
}
206+
207+
157208
private static class ParsedRule {
158209
final RuleTransformerModel model;
159210
final String fileName;
@@ -163,4 +214,4 @@ private static class ParsedRule {
163214
this.fileName = fileName;
164215
}
165216
}
166-
}
217+
}

‎data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleEvaluatorResult.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,21 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
48
*/
9+
510
package org.opensearch.dataprepper.pipeline.parser.rule;
611

712
import lombok.AllArgsConstructor;
813
import lombok.Builder;
914
import lombok.Getter;
1015
import org.opensearch.dataprepper.pipeline.parser.transformer.PipelineTemplateModel;
1116

17+
import java.util.List;
18+
1219
@Builder(setterPrefix = "with")
1320
@Getter
1421
@AllArgsConstructor
@@ -20,6 +27,8 @@ public class RuleEvaluatorResult {
2027

2128
private PipelineTemplateModel pipelineTemplateModel;
2229

30+
private List<String> functionProviders;
31+
2332
public RuleEvaluatorResult() {
2433

2534
}

‎data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleFileEvaluation.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,16 @@
44
import lombok.Builder;
55
import lombok.Data;
66

7+
import java.util.List;
8+
79
@Builder(setterPrefix = "with")
810
@AllArgsConstructor
911
@Data
1012
public class RuleFileEvaluation {
1113
private Boolean result;
1214
private String ruleFileName;
1315
private String pluginName;
16+
private List<String> functionProviders;
1417

1518
public RuleFileEvaluation() {
1619

‎data-prepper-pipeline-parser/src/main/java/org/opensearch/dataprepper/pipeline/parser/rule/RuleTransformerModel.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
/*
22
* Copyright OpenSearch Contributors
33
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
48
*/
9+
510
package org.opensearch.dataprepper.pipeline.parser.rule;
611

712
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -22,13 +27,17 @@ public class RuleTransformerModel {
2227
@JsonProperty("plugin_name")
2328
private String pluginName;
2429

30+
@JsonProperty("function_providers")
31+
private List<String> functionProviders;
32+
2533
public RuleTransformerModel() {
2634
}
2735

2836
@Override
2937
public String toString() {
3038
return "RuleConfiguration{" +
3139
"applyWhen=" + applyWhen +
32-
"\npluginName="+ pluginName +'}';
40+
"\nfunctionProviders=" + functionProviders +
41+
"\npluginName=" + pluginName + '}';
3342
}
34-
}
43+
}

0 commit comments

Comments
 (0)