Test framework support for loading plugin instances

data-prepper-plugins/drop-events-processor/build.gradle

@@ -11,6 +11,7 @@ dependencies {
     implementation project(':data-prepper-api')
     implementation project(':data-prepper-plugins:common')
     implementation 'com.fasterxml.jackson.core:jackson-databind'
+    testImplementation project(':data-prepper-test:plugin-test-framework')
 }
 
 jacocoTestCoverageVerification {

data-prepper-plugins/drop-events-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/drop/DropEventsProcessorIT.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.plugins.processor.drop;
+
+import org.junit.jupiter.api.Test;
+import org.opensearch.dataprepper.model.event.Event;
+import org.opensearch.dataprepper.model.event.EventFactory;
+import org.opensearch.dataprepper.model.event.LogEventBuilder;
+import org.opensearch.dataprepper.model.processor.Processor;
+import org.opensearch.dataprepper.model.record.Record;
+import org.opensearch.dataprepper.test.plugins.DataPrepperPluginTest;
+import org.opensearch.dataprepper.test.plugins.PluginConfigurationFile;
+import org.opensearch.dataprepper.test.plugins.junit.BaseDataPrepperPluginStandardTestSuite;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+@DataPrepperPluginTest(pluginName = "drop_events", pluginType = Processor.class)
+class DropEventsProcessorIT extends BaseDataPrepperPluginStandardTestSuite {
+    @Test
+    void drops_records_when_value_is_empty_string(
+            @PluginConfigurationFile("drop_when_value_is_empty_string.yaml") final Processor<Record<Event>, Record<Event>> objectUnderTest,
+            final EventFactory eventFactory) {
+
+        final List<Event> allEvents = new LinkedList<>();
+        final List<Event> expectedEventsAfterProcessor = new LinkedList<>();
+        for(int i = 0; i < 5; i++) {
+
+            final boolean shouldKeepEvent = i % 2 == 0;
+
+            final String keyValue = shouldKeepEvent ? UUID.randomUUID().toString() : "";
+            final Event event = eventFactory.eventBuilder(LogEventBuilder.class)
+                    .withData(Map.of(
+                            "my_key", keyValue
+                            ,"some_other_key", UUID.randomUUID().toString()
+                    ))
+                    .build();
+
+            allEvents.add(event);
+            if(shouldKeepEvent) {
+                expectedEventsAfterProcessor.add(event);
+            }
+        }
+
+        final List<Record<Event>> inputRecords = allEvents.stream()
+                .map(Record::new)
+                .collect(Collectors.toList());
+
+        final Collection<Record<Event>> outputRecords = objectUnderTest.execute(inputRecords);
+
+        assertThat(outputRecords, notNullValue());
+
+        final List<Event> outputEvents = outputRecords.stream().map(Record::getData).collect(Collectors.toList());
+
+        assertThat(outputEvents, equalTo(expectedEventsAfterProcessor));
+    }
+}

data-prepper-plugins/drop-events-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/drop/drop_when_value_is_empty_string.yaml

@@ -0,0 +1,8 @@
+test-pipeline:
+  source:
+    unused:
+  processor:
+    - drop_events:
+        drop_when: /my_key == ""
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_no_captures_with_existing_and_non_existing_key.yaml

@@ -0,0 +1,11 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{SYSLOGBASE}"]
+          bad_key: ["%{SYSLOGBASE}"]
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_keep_empty_captures_true.yaml

@@ -0,0 +1,11 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{COMMONAPACHELOG}"]
+        keep_empty_captures: true
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_named_captures_only_false.yaml

@@ -0,0 +1,11 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{GREEDYDATA} %{IPORHOST:host} %{NUMBER}"]
+        named_captures_only: false
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_named_captures_syntax.yaml

@@ -0,0 +1,10 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{GREEDYDATA:greedy_data} (?<mynumber>\\d\\d\\d-\\d\\d\\d-\\d\\d\\d)"]
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/match_with_no_captures_and_tags.yaml

@@ -0,0 +1,13 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{GREEDYDATA:greedy_data} (?<mynumber>\\d\\d\\d-\\d\\d\\d-\\d\\d\\d)"]
+        tags_on_match_failure:
+          - some_test_tag_1
+          - some_test_tag_2
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/multiple_match_with_break_on_match_false.yaml

@@ -0,0 +1,12 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{COMMONAPACHELOG}"]
+          extra_field: ["%{GREEDYDATA} %{IPORHOST:host}"]
+        break_on_match: false
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/pattern_definitions.yaml

@@ -0,0 +1,12 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{GREEDYDATA:greedy_data} %{CUSTOMPHONENUMBERPATTERN:my_number}"]
+        pattern_definitions:
+          CUSTOMPHONENUMBERPATTERN: "\\d\\d\\d-\\d\\d\\d-\\d\\d\\d"
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/patterns_dir_with_custom_patterns_files_glob.yaml

@@ -0,0 +1,14 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message:
+            - "My phone number is %{CUSTOMPHONENUMBERPATTERN:my_number}"
+        patterns_directories:
+          - ./src/test/resources/test_patterns
+        patterns_files_glob: "*1.txt"
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/patterns_dir_with_default_patterns_files_glob.yaml

@@ -0,0 +1,13 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message:
+            - "My birthday is %{CUSTOMBIRTHDAYPATTERN:my_birthday} and my phone number is %{CUSTOMPHONENUMBERPATTERN:my_number}"
+        patterns_directories:
+          - ./src/test/resources/test_patterns
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/single_match_multiple_pattern_with_break_on_match_false.yaml

@@ -0,0 +1,13 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message:
+            - "%{COMMONAPACHELOG}"
+            - "%{IPORHOST:custom_client_field}"
+        break_on_match: false
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/single_match_single_pattern.yaml

@@ -0,0 +1,10 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message: ["%{COMMONAPACHELOG}"]
+
+  sink:
+    - unused:

data-prepper-plugins/grok-processor/src/test/resources/org/opensearch/dataprepper/plugins/processor/grok/single_match_type_conversion.yaml

@@ -0,0 +1,11 @@
+grok-pipeline:
+  source:
+    unused:
+  processor:
+    - grok:
+        match:
+          message:
+            - "\"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response:int} (?:%{NUMBER:bytes:float}|-)"
+
+  sink:
+    - unused:

data-prepper-plugins/parse-json-processor/build.gradle

@@ -17,6 +17,7 @@ dependencies {
     implementation libs.parquet.common
     testImplementation project(':data-prepper-test:test-common')
     testImplementation project(':data-prepper-test:test-event')
+    testImplementation project(':data-prepper-test:plugin-test-framework')
     testImplementation testLibs.slf4j.simple
 }