diff --git a/data-prepper-plugins/http-sink/build.gradle b/data-prepper-plugins/http-sink/build.gradle index dbfe13c70e..447eb7c177 100644 --- a/data-prepper-plugins/http-sink/build.gradle +++ b/data-prepper-plugins/http-sink/build.gradle @@ -28,6 +28,8 @@ dependencies { implementation project(':data-prepper-plugins:parse-json-processor') implementation 'software.amazon.awssdk:sts' testImplementation project(':data-prepper-test:test-common') + testImplementation project(':data-prepper-test:test-event') + testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' } test { @@ -57,11 +59,10 @@ task integrationTest(type: Test) { useJUnitPlatform() classpath = sourceSets.integrationTest.runtimeClasspath - systemProperty 'tests.http.sink.http.endpoint', System.getProperty('tests.http.sink.http.endpoint') - systemProperty 'tests.http.sink.region', System.getProperty('tests.http.sink.region') - systemProperty 'tests.http.sink.bucket', System.getProperty('tests.http.sink.bucket') + systemProperty 'tests.aws.region', System.getProperty('tests.aws.region') + systemProperty 'tests.aws.role', System.getProperty('tests.aws.role') filter { includeTestsMatching '*IT' } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkIT.java b/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkIT.java new file mode 100644 index 0000000000..6f57837c92 --- /dev/null +++ b/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkIT.java @@ -0,0 +1,582 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.sun.net.httpserver.HttpServer; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Timer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoSettings; +import static org.awaitility.Awaitility.await; +import org.mockito.quality.Strictness; +import org.opensearch.dataprepper.aws.api.AwsConfig; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.core.pipeline.Pipeline; +import org.opensearch.dataprepper.event.TestEventFactory; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.codec.OutputCodec; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.codec.json.JsonOutputCodec; +import org.opensearch.dataprepper.plugins.codec.json.JsonOutputCodecConfig; +import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; +import org.opensearch.dataprepper.plugins.sink.http.configuration.ThresholdOptions; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +@MockitoSettings(strictness = Strictness.LENIENT) + +class HttpSinkIT { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final int NUM_RECORDS = 10; + private static final int PORT = 8888; + private static final int SIGV4_PORT = 8889; + private HttpServer server; + private HttpServer sigv4Server; + private String serverUrl; + private String sigv4ServerUrl; + + private List receivedRequests; + private List sigv4ReceivedRequests; + @Mock + private PluginFactory pluginFactory; + + @Mock + private Pipeline dlqPipeline; + + @Mock + private PipelineDescription pipelineDescription; + + @Mock + private AwsCredentialsSupplier awsCredentialsSupplier; + + @Mock + private PluginSetting pluginSetting; + + @Mock + private PluginMetrics pluginMetrics; + + @Mock + private Counter eventsSuccessCounter; + @Mock + private Counter eventsFailedCounter; + @Mock + private Counter requestsSuccessCounter; + @Mock + private Counter requestsFailedCounter; + @Mock + private Counter requestRetriesCounter; + @Mock + private DistributionSummary summary; + @Mock + private Timer timer; + @Mock + AwsConfig awsConfig; + + private String awsRegion; + private String awsRole; + + @Mock + private HttpSinkConfiguration httpSinkConfiguration; + + @BeforeEach + void setUp() throws Exception { + OBJECT_MAPPER.registerModule(new JavaTimeModule()); + receivedRequests = new ArrayList<>(); + sigv4ReceivedRequests = new ArrayList<>(); + server = HttpServer.create(new InetSocketAddress(PORT), 0); + serverUrl = "http://localhost:" + PORT; + server.start(); + + sigv4Server = HttpServer.create(new InetSocketAddress(SIGV4_PORT), 0); + sigv4ServerUrl = "http://localhost:" + SIGV4_PORT; + sigv4Server.start(); + + timer = mock(Timer.class); + summary = mock(DistributionSummary.class); + + eventsSuccessCounter = mock(Counter.class); + eventsFailedCounter = mock(Counter.class); + requestsSuccessCounter = mock(Counter.class); + requestsFailedCounter = mock(Counter.class); + pluginMetrics = mock(PluginMetrics.class); + + when(pluginMetrics.counter(eq("sinkRequestsSucceeded"))).thenReturn(requestsSuccessCounter); + when(pluginMetrics.counter(eq("sinkRequestsFailed"))).thenReturn(requestsFailedCounter); + when(pluginMetrics.counter(eq("sinkEventsSucceeded"))).thenReturn(eventsSuccessCounter); + when(pluginMetrics.counter(eq("sinkEventsFailed"))).thenReturn(eventsFailedCounter); + when(pluginMetrics.counter(eq("sinkRetries"))).thenReturn(requestRetriesCounter); + + when(pluginMetrics.summary(any(String.class))).thenReturn(summary); + when(pluginMetrics.timer(any(String.class))).thenReturn(timer); + + awsConfig = mock(AwsConfig.class); + awsRegion = System.getProperty("tests.aws.region"); + when(awsConfig.getAwsRegion()).thenReturn(Region.of(awsRegion)); + awsRole = System.getProperty("tests.aws.role"); + when(awsConfig.getAwsStsRoleArn()).thenReturn(awsRole); + + pluginFactory = mock(PluginFactory.class); + JsonOutputCodec jsonCodec = new JsonOutputCodec(new JsonOutputCodecConfig()); + when(pluginFactory.loadPlugin(eq(OutputCodec.class), any())).thenReturn(jsonCodec); + pluginSetting = mock(PluginSetting.class); + when(pluginSetting.getPipelineName()).thenReturn("test-pipeline"); + when(pluginSetting.getName()).thenReturn("name"); + pipelineDescription = mock(PipelineDescription.class); + awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); + when(awsCredentialsSupplier.getProvider(any())).thenAnswer(options -> DefaultCredentialsProvider.create()); + + when(pipelineDescription.getPipelineName()).thenReturn("test-pipeline"); + } + + HttpSink createObjectUnderTest(HttpSinkConfiguration config) throws IOException { + return new HttpSink(pluginSetting, config, pluginFactory, pipelineDescription, + null, awsCredentialsSupplier, pluginMetrics); + } + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + if (sigv4Server != null) { + sigv4Server.stop(0); + } + receivedRequests.clear(); + sigv4ReceivedRequests.clear(); + } + + @Test + void testHttpSink_withSuccessResponseWithEventCountThreshold() throws Exception { + server.createContext("/success", exchange -> { + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + receivedRequests.add(body); + exchange.sendResponseHeaders(200, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfig(serverUrl + "/success", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + for (int i = 0; i < NUM_RECORDS; i++) { + Event event = createEvent(Map.of("key", "value" + i)); + records.add(new Record<>(event)); + } + sink.doOutput(records); + await().atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> { + assertEquals(NUM_RECORDS, receivedRequests.size()); + for (int i = 0; i < NUM_RECORDS; i++) { + assertTrue(receivedRequests.get(i).contains("value"+i)); + } + }); + verify(requestsSuccessCounter, times(NUM_RECORDS)).increment(1); + verify(eventsSuccessCounter, times(NUM_RECORDS)).increment(1); + } + + @Test + void testHttpSink_withSuccessResponseWithMaxSizeThresholdAndFlushTimeOut() throws Exception { + server.createContext("/success", exchange -> { + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + receivedRequests.add(body); + exchange.sendResponseHeaders(200, 0); + exchange.close(); + }); + + String json = "{\"max_request_size\": \"20b\", \"flush_timeout\": \"PT10S\"}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfig(serverUrl + "/success", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + for (int i = 0; i < NUM_RECORDS; i++) { + Event event = createEvent(Map.of("test_key", "test_value" + i)); + records.add(new Record<>(event)); + } + sink.doOutput(records); + await().atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> { + sink.doOutput(Collections.emptyList()); + assertEquals(NUM_RECORDS, receivedRequests.size()); + for (int i = 0; i < NUM_RECORDS; i++) { + assertTrue(receivedRequests.get(i).contains("value"+i)); + } + }); + verify(requestsSuccessCounter, times(NUM_RECORDS)).increment(1); + verify(eventsSuccessCounter, times(NUM_RECORDS)).increment(1); + } + + + Event createEvent(Map data) { + return TestEventFactory.getTestEventFactory().eventBuilder(EventBuilder.class) + .withData(data) + .withEventType("event") + .build(); + } + + @Test + void testHttpSink_withServerError() throws Exception { + server.createContext("/error", exchange -> { + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + receivedRequests.add(body); + exchange.sendResponseHeaders(500, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfig(serverUrl + "/error", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + Event event = createEvent(Map.of("test_key", "test_value")); + records.add(new Record<>(event)); + + + sink.doOutput(records); + + await().atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> { + verify(requestRetriesCounter, times(3)).increment(1); + }); + verify(requestsFailedCounter, times(1)).increment(1); + verify(eventsFailedCounter, times(1)).increment(1); + } + + @Test + void testHttpSink_withBadRequest() throws Exception { + server.createContext("/badrequest", exchange -> { + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + receivedRequests.add(body); + exchange.sendResponseHeaders(401, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfig(serverUrl + "/error", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + Event event = createEvent(Map.of("test_key", "test_value")); + records.add(new Record<>(event)); + sink.doOutput(records); + + await().atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> { + + verify(requestsFailedCounter, times(1)).increment(1); + verify(eventsFailedCounter, times(1)).increment(1); + }); + + verify(requestRetriesCounter, times(0)).increment(1); + } + + private HttpSinkConfiguration createConfig(String url) { + return createConfig(url, new ThresholdOptions()); + } + + private HttpSinkConfiguration createConfig(String url, ThresholdOptions thresholdOptions) { + HttpSinkConfiguration config = new HttpSinkConfiguration(); + Map codecSettings = new HashMap<>(); + PluginModel codecModel = new PluginModel("json", codecSettings); + + try { + java.lang.reflect.Field urlField = HttpSinkConfiguration.class.getDeclaredField("url"); + urlField.setAccessible(true); + urlField.set(config, url); + + java.lang.reflect.Field codecField = HttpSinkConfiguration.class.getDeclaredField("codec"); + codecField.setAccessible(true); + codecField.set(config, codecModel); + + java.lang.reflect.Field thresholdField = HttpSinkConfiguration.class.getDeclaredField("thresholdOptions"); + thresholdField.setAccessible(true); + thresholdField.set(config, thresholdOptions); + + java.lang.reflect.Field retryIntervalField = HttpSinkConfiguration.class.getDeclaredField("httpRetryInterval"); + retryIntervalField.setAccessible(true); + retryIntervalField.set(config, Duration.ofSeconds(3)); + + java.lang.reflect.Field serviceNameField = HttpSinkConfiguration.class.getDeclaredField("awsSigv4ServiceName"); + serviceNameField.setAccessible(true); + serviceNameField.set(config, "test"); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return config; + } + + private HttpSinkConfiguration createConfigWithSigV4(String url, ThresholdOptions thresholdOptions) { + HttpSinkConfiguration config = new HttpSinkConfiguration(); + Map codecSettings = new HashMap<>(); + PluginModel codecModel = new PluginModel("json", codecSettings); + + try { + java.lang.reflect.Field urlField = HttpSinkConfiguration.class.getDeclaredField("url"); + urlField.setAccessible(true); + urlField.set(config, url); + + java.lang.reflect.Field codecField = HttpSinkConfiguration.class.getDeclaredField("codec"); + codecField.setAccessible(true); + codecField.set(config, codecModel); + + java.lang.reflect.Field thresholdField = HttpSinkConfiguration.class.getDeclaredField("thresholdOptions"); + thresholdField.setAccessible(true); + thresholdField.set(config, thresholdOptions); + + java.lang.reflect.Field awsField = HttpSinkConfiguration.class.getDeclaredField("awsConfig"); + awsField.setAccessible(true); + awsField.set(config, awsConfig); + + java.lang.reflect.Field retryIntervalField = HttpSinkConfiguration.class.getDeclaredField("httpRetryInterval"); + retryIntervalField.setAccessible(true); + retryIntervalField.set(config, Duration.ofSeconds(3)); + + java.lang.reflect.Field serviceNameField = HttpSinkConfiguration.class.getDeclaredField("awsSigv4ServiceName"); + serviceNameField.setAccessible(true); + serviceNameField.set(config, "test"); + + + + } catch (Exception e) { + throw new RuntimeException(e); + } + + return config; + } + + @Test + void testHttpSink_withSuccessResponseWithEventCountThreshold_SigV4() throws Exception { + sigv4Server.createContext("/success", exchange -> { + String authHeader = exchange.getRequestHeaders().getFirst("Authorization"); + if (authHeader == null || !authHeader.contains("AWS4-HMAC-SHA256")) { + exchange.sendResponseHeaders(401, 0); + exchange.close(); + return; + } + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + sigv4ReceivedRequests.add(body); + exchange.sendResponseHeaders(200, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfigWithSigV4(sigv4ServerUrl + "/success", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + for (int i = 0; i < NUM_RECORDS; i++) { + Event event = createEvent(Map.of("key", "value" + i)); + records.add(new Record<>(event)); + } + sink.doOutput(records); + await().atMost(Duration.ofSeconds(30)) + .untilAsserted(() -> { + assertEquals(NUM_RECORDS, sigv4ReceivedRequests.size()); + for (int i = 0; i < NUM_RECORDS; i++) { + assertTrue(sigv4ReceivedRequests.get(i).contains("value"+i)); + } + }); + verify(requestsSuccessCounter, times(NUM_RECORDS)).increment(1); + verify(eventsSuccessCounter, times(NUM_RECORDS)).increment(1); + } + + @Test + void testHttpSink_withSuccessResponseWithMaxSizeThreshold_SigV4() throws Exception { + sigv4Server.createContext("/success", exchange -> { + String authHeader = exchange.getRequestHeaders().getFirst("Authorization"); + if (authHeader == null || !authHeader.contains("AWS4-HMAC-SHA256")) { + exchange.sendResponseHeaders(401, 0); + exchange.close(); + return; + } + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + sigv4ReceivedRequests.add(body); + exchange.sendResponseHeaders(200, 0); + exchange.close(); + }); + + String json = "{\"max_request_size\": \"20b\", \"flush_timeout\": \"PT10S\"}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfigWithSigV4(sigv4ServerUrl + "/success", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + for (int i = 0; i < NUM_RECORDS; i++) { + Event event = createEvent(Map.of("test_key", "test_value" + i)); + records.add(new Record<>(event)); + } + sink.doOutput(records); + await().atMost(Duration.ofSeconds(30)) + .untilAsserted(() -> { + sink.doOutput(Collections.emptyList()); + assertEquals(NUM_RECORDS, sigv4ReceivedRequests.size()); + for (int i = 0; i < NUM_RECORDS; i++) { + assertTrue(sigv4ReceivedRequests.get(i).contains("value"+i)); + } + }); + verify(requestsSuccessCounter, times(NUM_RECORDS)).increment(1); + verify(eventsSuccessCounter, times(NUM_RECORDS)).increment(1); + } + + @Test + void testHttpSink_withServerError_SigV4() throws Exception { + sigv4Server.createContext("/error", exchange -> { + String authHeader = exchange.getRequestHeaders().getFirst("Authorization"); + if (authHeader == null || !authHeader.contains("AWS4-HMAC-SHA256")) { + exchange.sendResponseHeaders(401, 0); + exchange.close(); + return; + } + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + sigv4ReceivedRequests.add(body); + exchange.sendResponseHeaders(500, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfigWithSigV4(sigv4ServerUrl + "/error", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + Event event = createEvent(Map.of("test_key", "test_value")); + records.add(new Record<>(event)); + + sink.doOutput(records); + + await().atMost(Duration.ofSeconds(30)) + .untilAsserted(() -> { + verify(requestRetriesCounter, times(3)).increment(1); + }); + verify(requestsFailedCounter, times(1)).increment(1); + verify(eventsFailedCounter, times(1)).increment(1); + } + + @Test + void testHttpSink_withDLQPipeline() throws Exception { + dlqPipeline = mock(Pipeline.class); + server.createContext("/badrequest", exchange -> { + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + receivedRequests.add(body); + exchange.sendResponseHeaders(401, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfig(serverUrl + "/error", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + sink.setDlqPipeline(dlqPipeline); + + List> records = new ArrayList<>(); + Event event = createEvent(Map.of("test_key", "test_value")); + records.add(new Record<>(event)); + sink.doOutput(records); + + await().atMost(Duration.ofSeconds(60)) + .untilAsserted(() -> { + + verify(requestsFailedCounter, times(1)).increment(1); + verify(eventsFailedCounter, times(1)).increment(1); + }); + + verify(requestRetriesCounter, times(0)).increment(1); + verify(dlqPipeline, times(1)).sendEvents(any()); + } + + @Test + public void testToVerifyLackOfCredentialsResultInFailure() throws Exception { + AwsCredentialsProvider provider = mock(AwsCredentialsProvider.class); + when(awsCredentialsSupplier.getProvider(any())).thenReturn(provider); + sigv4Server.createContext("/success", exchange -> { + String authHeader = exchange.getRequestHeaders().getFirst("Authorization"); + if (authHeader == null || !authHeader.contains("AWS4-HMAC-SHA256")) { + exchange.sendResponseHeaders(401, 0); + exchange.close(); + return; + } + InputStream is = exchange.getRequestBody(); + String body = new String(is.readAllBytes(), StandardCharsets.UTF_8); + sigv4ReceivedRequests.add(body); + exchange.sendResponseHeaders(200, 0); + exchange.close(); + }); + + String json = "{\"max_events\": 1}"; + ThresholdOptions thresholdOptions = OBJECT_MAPPER.readValue(json, ThresholdOptions.class); + HttpSinkConfiguration config = createConfigWithSigV4(sigv4ServerUrl + "/success", thresholdOptions); + HttpSink sink = createObjectUnderTest(config); + sink.doInitialize(); + + List> records = new ArrayList<>(); + for (int i = 0; i < NUM_RECORDS; i++) { + Event event = createEvent(Map.of("key", "value" + i)); + records.add(new Record<>(event)); + } + sink.doOutput(records); + verify(requestsFailedCounter, times(NUM_RECORDS)).increment(1); + } +} diff --git a/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkServiceIT.java b/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkServiceIT.java index 2095edcb48..896ee9961a 100644 --- a/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkServiceIT.java +++ b/data-prepper-plugins/http-sink/src/integrationTest/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkServiceIT.java @@ -1,7 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http; import com.fasterxml.jackson.core.JsonProcessingException; @@ -9,11 +15,10 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import io.micrometer.core.instrument.Counter; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.impl.classic.HttpClients; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; +import org.opensearch.dataprepper.common.sink.SinkMetrics; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.codec.OutputCodec; import org.opensearch.dataprepper.model.configuration.PipelineDescription; @@ -24,13 +29,11 @@ import org.opensearch.dataprepper.model.log.JacksonLog; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.accumulator.BufferFactory; -import org.opensearch.dataprepper.plugins.accumulator.InMemoryBufferFactory; +import org.opensearch.dataprepper.model.sink.OutputCodecContext; import org.opensearch.dataprepper.plugins.codec.json.NdjsonOutputCodec; import org.opensearch.dataprepper.plugins.codec.json.NdjsonOutputConfig; import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; import org.opensearch.dataprepper.plugins.sink.http.configuration.ThresholdOptions; -import org.opensearch.dataprepper.plugins.sink.http.dlq.DlqPushHandler; import org.opensearch.dataprepper.plugins.sink.http.service.HttpSinkService; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; @@ -40,10 +43,13 @@ import java.util.LinkedList; import java.util.UUID; -import static org.mockito.Mockito.mock; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.sink.http.service.HttpSinkService.HTTP_SINK_RECORDS_SUCCESS_COUNTER; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; + public class HttpSinkServiceIT { @@ -55,43 +61,49 @@ public class HttpSinkServiceIT { private String config = " url: {0}\n" + - " http_method: POST\n" + - " auth_type: {1}\n" + " codec:\n" + " json:\n" + - " insecure_skip_verify: true\n" + " threshold:\n" + - " event_count: 1"; + " max_events: 1"; private HttpSinkConfiguration httpSinkConfiguration; - private BufferFactory bufferFactory; - - private DlqPushHandler dlqPushHandler; - private PluginMetrics pluginMetrics; private PluginSetting pluginSetting; - @BeforeEach - void setUp() throws JsonProcessingException{ - this.urlString = System.getProperty("tests.http.sink.http.endpoint"); - String[] values = { urlString,"unauthenticated"}; - final String configYaml = MessageFormat.format(config, values); - this.httpSinkConfiguration = objectMapper.readValue(configYaml, HttpSinkConfiguration.class); - } + @Mock private PipelineDescription pipelineDescription; + @Mock + private SinkMetrics sinkMetrics; + @Mock + private HttpSinkSender httpSinkSender; private PluginFactory pluginFactory; @Mock private Counter httpSinkRecordsSuccessCounter; + @Mock + private HttpEndpointResponse httpEndpointResponse; + @Mock NdjsonOutputConfig ndjsonOutputConfig; + @BeforeEach + void setUp() throws JsonProcessingException{ + this.urlString = System.getProperty("tests.http.sink.http.endpoint"); + final String configYaml = MessageFormat.format(config, (Object) urlString, "unauthenticated"); + this.httpSinkConfiguration = objectMapper.readValue(configYaml, HttpSinkConfiguration.class); + this.httpSinkSender = mock(HttpSinkSender.class); + httpEndpointResponse = mock(HttpEndpointResponse.class); + when(httpEndpointResponse.getStatusCode()).thenReturn(200); + when(httpSinkSender.send(any())).thenReturn(httpEndpointResponse); + sinkMetrics = mock(SinkMetrics.class); + } + public HttpSinkService createHttpSinkServiceUnderTest() throws NoSuchFieldException, IllegalAccessException { this.pipelineDescription = mock(PipelineDescription.class); this.pluginFactory = mock(PluginFactory.class); @@ -99,34 +111,24 @@ public HttpSinkService createHttpSinkServiceUnderTest() throws NoSuchFieldExcept this.pluginMetrics = mock(PluginMetrics.class); this.pluginSetting = mock(PluginSetting.class); - when(pluginMetrics.counter(HTTP_SINK_RECORDS_SUCCESS_COUNTER)).thenReturn(httpSinkRecordsSuccessCounter); + when(pluginMetrics.counter(eq("sinkEventsSucceeded"))).thenReturn(httpSinkRecordsSuccessCounter); when(pipelineDescription.getPipelineName()).thenReturn("http-plugin"); - ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfiguration.getThresholdOptions(),"eventCollectTimeOut", Duration.ofNanos(1)); + ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfiguration.getThresholdOptions(),"flushTimeout", Duration.ofNanos(1)); final PluginModel codecConfiguration = httpSinkConfiguration.getCodec(); final PluginSetting codecPluginSettings = new PluginSetting(codecConfiguration.getPluginName(), codecConfiguration.getPluginSettings()); this.ndjsonOutputConfig = mock(NdjsonOutputConfig.class); codec = new NdjsonOutputCodec(ndjsonOutputConfig); - this.bufferFactory = new InMemoryBufferFactory(); - this.dlqPushHandler = new DlqPushHandler(httpSinkConfiguration.getDlqFile(), pluginFactory, - "bucket", - "arn", "region", - "keypath"); - - HttpClientBuilder httpClientBuilder = HttpClients.custom(); - - return new HttpSinkService( - httpSinkConfiguration, - bufferFactory, - dlqPushHandler, - codecPluginSettings, - null, - httpClientBuilder, - pluginMetrics, - pluginSetting, - codec, - null); + OutputCodecContext codecContext = new OutputCodecContext(); + doAnswer((args)-> { + int count = (int)args.getArgument(0); + httpSinkRecordsSuccessCounter.increment(count); + return null; + }).when(sinkMetrics).incrementEventsSuccessCounter(any(Integer.class)); + + return new HttpSinkService(httpSinkConfiguration, sinkMetrics, httpSinkSender, pipelineDescription, + codec, codecContext); } private Collection> setEventQueue(final int records) { diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AuthenticationDecorator.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AuthenticationDecorator.java new file mode 100644 index 0000000000..37097380fc --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AuthenticationDecorator.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http; + +import com.linecorp.armeria.common.HttpRequest; + +import java.util.List; +import java.util.Map; + +public interface AuthenticationDecorator { + HttpRequest buildRequest(String url, byte[] payload, Map> customHeaders); +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsAuthenticationDecorator.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsAuthenticationDecorator.java new file mode 100644 index 0000000000..533f8aef1f --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsAuthenticationDecorator.java @@ -0,0 +1,107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http; + +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.RequestHeadersBuilder; +import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.aws.api.AwsConfig; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.signer.Aws4Signer; +import software.amazon.awssdk.auth.signer.params.Aws4SignerParams; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.regions.Region; + +import javax.annotation.Nonnull; +import java.net.URI; +import java.util.List; +import java.util.Map; + +public class AwsAuthenticationDecorator implements AuthenticationDecorator { + private static final String CONTENT_SHA256_HEADER = "x-amz-content-sha256"; + private static final String CONTENT_SHA256_VALUE = "required"; + + private final Aws4Signer signer = Aws4Signer.create(); + private final AwsCredentialsProvider credentialsProvider; + private final Region region; + private final String serviceName; + + public AwsAuthenticationDecorator(@Nonnull final AwsCredentialsSupplier awsCredentialsSupplier, + @Nonnull final AwsConfig awsConfig, + @Nonnull final String serviceName) { + this.region = awsConfig.getAwsRegion(); + this.serviceName = serviceName; + this.credentialsProvider = awsCredentialsSupplier.getProvider(convertToCredentialOptions(awsConfig)); + } + + private static AwsCredentialsOptions convertToCredentialOptions(final AwsConfig awsConfig) { + return AwsCredentialsOptions.builder() + .withRegion(awsConfig.getAwsRegion()) + .withStsRoleArn(awsConfig.getAwsStsRoleArn()) + .withStsExternalId(awsConfig.getAwsStsExternalId()) + .withStsHeaderOverrides(awsConfig.getAwsStsHeaderOverrides()) + .build(); + } + + @Override + public HttpRequest buildRequest(final String url, final byte[] payload, final Map> customHeaders) { + final SdkHttpFullRequest sdkRequest = createSdkHttpRequest(url, payload, customHeaders); + final SdkHttpFullRequest signedRequest = sign(sdkRequest); + return toArmeriaRequest(signedRequest, payload); + } + + private SdkHttpFullRequest createSdkHttpRequest(final String url, final byte[] payload, + final Map> customHeaders) { + final SdkHttpFullRequest.Builder builder = SdkHttpFullRequest.builder() + .method(SdkHttpMethod.POST) + .uri(URI.create(url)) + .contentStreamProvider(() -> SdkBytes.fromByteArray(payload).asInputStream()); + + if (customHeaders != null) { + customHeaders.forEach((key, values) -> + values.forEach(value -> builder.appendHeader(key, value)) + ); + } + return builder.build(); + } + + private SdkHttpFullRequest sign(final SdkHttpFullRequest request) { + final SdkHttpFullRequest requestWithHeader = request.toBuilder() + .putHeader(CONTENT_SHA256_HEADER, CONTENT_SHA256_VALUE) + .build(); + + return signer.sign(requestWithHeader, Aws4SignerParams.builder() + .signingRegion(region) + .signingName(serviceName) + .awsCredentials(credentialsProvider.resolveCredentials()) + .build()); + } + + private static HttpRequest toArmeriaRequest(final SdkHttpFullRequest sdkRequest, final byte[] payload) { + final RequestHeadersBuilder headersBuilder = RequestHeaders.builder() + .method(HttpMethod.POST) + .scheme(sdkRequest.getUri().getScheme()) + .path(sdkRequest.getUri().getRawPath()) + .authority(sdkRequest.getUri().getAuthority()); + + sdkRequest.headers().forEach((k, vList) -> + vList.forEach(v -> headersBuilder.add(k, v)) + ); + + return HttpRequest.of(headersBuilder.build(), HttpData.wrap(payload)); + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsRequestSigningApacheInterceptor.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsRequestSigningApacheInterceptor.java index 84b13a17c9..5cd060427c 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsRequestSigningApacheInterceptor.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/AwsRequestSigningApacheInterceptor.java @@ -1,19 +1,15 @@ /* - * Copyright OpenSearch Contributors. All Rights Reserved. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 * - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with - * the License. A copy of the License is located at + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. * - * http://aws.amazon.com/apache2.0 - * - * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions - * and limitations under the License. */ + package org.opensearch.dataprepper.plugins.sink.http; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.EntityDetails; @@ -145,8 +141,6 @@ public void process(final HttpRequest request, final EntityDetails entity, final requestBuilder.rawQueryParameters(nvpToMapParams(uriBuilder.getQueryParams())); requestBuilder.headers(headerArrayToMap(request.getHeaders())); - AWSCredentials credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); - ExecutionAttributes attributes = new ExecutionAttributes(); attributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, awsCredentialsProvider.resolveCredentials()); attributes.putAttribute(AwsSignerExecutionAttribute.SERVICE_SIGNING_NAME, service); @@ -230,4 +224,4 @@ private static Header[] mapToHeaderArray(final Map> mapHead } return headers; } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptor.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptor.java index f2961b4948..b95b41501c 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptor.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptor.java @@ -1,7 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http; import org.apache.hc.core5.http.EntityDetails; diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HTTPSink.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HTTPSink.java deleted file mode 100644 index 3f43df2755..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HTTPSink.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http; - -import org.apache.hc.client5.http.HttpRequestRetryStrategy; -import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.core5.util.TimeValue; -import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; -import org.opensearch.dataprepper.model.codec.OutputCodec; -import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; -import org.opensearch.dataprepper.model.plugin.PluginFactory; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.model.sink.AbstractSink; -import org.opensearch.dataprepper.model.sink.OutputCodecContext; -import org.opensearch.dataprepper.model.sink.Sink; -import org.opensearch.dataprepper.model.sink.SinkContext; -import org.opensearch.dataprepper.plugins.accumulator.BufferFactory; -import org.opensearch.dataprepper.plugins.accumulator.BufferTypeOptions; -import org.opensearch.dataprepper.plugins.accumulator.InMemoryBufferFactory; -import org.opensearch.dataprepper.plugins.accumulator.LocalFileBufferFactory; - -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.opensearch.dataprepper.plugins.sink.http.dlq.DlqPushHandler; -import org.opensearch.dataprepper.plugins.sink.http.service.HttpSinkAwsService; -import org.opensearch.dataprepper.plugins.sink.http.service.HttpSinkService; - -import org.opensearch.dataprepper.plugins.sink.http.service.WebhookService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.Collections; -import java.util.Objects; - -@DataPrepperPlugin(name = "http", pluginType = Sink.class, pluginConfigurationType = HttpSinkConfiguration.class) -public class HTTPSink extends AbstractSink> { - - private static final Logger LOG = LoggerFactory.getLogger(HTTPSink.class); - - private static final String BUCKET = "bucket"; - private static final String KEY_PATH = "key_path_prefix"; - - private WebhookService webhookService; - - private volatile boolean sinkInitialized; - - private final HttpSinkService httpSinkService; - - private final BufferFactory bufferFactory; - - private DlqPushHandler dlqPushHandler; - - private final OutputCodec codec; - - private final SinkContext sinkContext; - - @DataPrepperPluginConstructor - public HTTPSink(final PluginSetting pluginSetting, - final HttpSinkConfiguration httpSinkConfiguration, - final PluginFactory pluginFactory, - final PipelineDescription pipelineDescription, - final SinkContext sinkContext, - final AwsCredentialsSupplier awsCredentialsSupplier) { - super(pluginSetting); - this.sinkContext = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); - final PluginModel codecConfiguration = httpSinkConfiguration.getCodec(); - final PluginSetting codecPluginSettings = new PluginSetting(codecConfiguration.getPluginName(), - codecConfiguration.getPluginSettings()); - codecPluginSettings.setPipelineName(pipelineDescription.getPipelineName()); - this.sinkInitialized = Boolean.FALSE; - if (httpSinkConfiguration.getBufferType().equals(BufferTypeOptions.LOCALFILE)) { - this.bufferFactory = new LocalFileBufferFactory(); - } else { - this.bufferFactory = new InMemoryBufferFactory(); - } - - this.dlqPushHandler = new DlqPushHandler(httpSinkConfiguration.getDlqFile(), pluginFactory, - String.valueOf(httpSinkConfiguration.getDlqPluginSetting().get(BUCKET)), - httpSinkConfiguration.getDlqStsRoleARN() - ,httpSinkConfiguration.getDlqStsRegion(), - String.valueOf(httpSinkConfiguration.getDlqPluginSetting().get(KEY_PATH))); - - final HttpRequestRetryStrategy httpRequestRetryStrategy = new DefaultHttpRequestRetryStrategy(httpSinkConfiguration.getMaxUploadRetries(), - TimeValue.of(httpSinkConfiguration.getHttpRetryInterval())); - if((!httpSinkConfiguration.isInsecure()) && (httpSinkConfiguration.isHttpUrl())){ - throw new InvalidPluginConfigurationException ("Cannot configure http url with insecure as false"); - } - - final HttpClientBuilder httpClientBuilder = HttpClients.custom() - .setRetryStrategy(httpRequestRetryStrategy); - - if(Objects.nonNull(httpSinkConfiguration.getWebhookURL())) - this.webhookService = new WebhookService(httpSinkConfiguration.getWebhookURL(), - httpClientBuilder,pluginMetrics,httpSinkConfiguration); - - if(httpSinkConfiguration.isAwsSigv4() && httpSinkConfiguration.isValidAWSUrl()){ - HttpSinkAwsService.attachSigV4(httpSinkConfiguration, httpClientBuilder, awsCredentialsSupplier); - } - this.codec = pluginFactory.loadPlugin(OutputCodec.class, codecPluginSettings); - this.httpSinkService = new HttpSinkService( - httpSinkConfiguration, - bufferFactory, - dlqPushHandler, - codecPluginSettings, - webhookService, - httpClientBuilder, - pluginMetrics, - pluginSetting, - codec, - OutputCodecContext.fromSinkContext(sinkContext)); - } - - @Override - public boolean isReady() { - return sinkInitialized; - } - - @Override - public void doInitialize() { - try { - doInitializeInternal(); - } catch (InvalidPluginConfigurationException e) { - LOG.error("Invalid plugin configuration, Hence failed to initialize http-sink plugin."); - this.shutdown(); - throw e; - } catch (Exception e) { - LOG.error("Failed to initialize http-sink plugin."); - this.shutdown(); - throw e; - } - } - - private void doInitializeInternal() { - sinkInitialized = Boolean.TRUE; - } - - /** - * @param records Records to be output - */ - @Override - public void doOutput(final Collection> records) { - if (records.isEmpty()) { - return; - } - httpSinkService.output(records); - } -} \ No newline at end of file diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpEndPointResponse.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpEndpointResponse.java similarity index 76% rename from data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpEndPointResponse.java rename to data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpEndpointResponse.java index 1247a5b9e5..88261290ed 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpEndPointResponse.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpEndpointResponse.java @@ -1,15 +1,21 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http; -public class HttpEndPointResponse { +public class HttpEndpointResponse { private String url; private int statusCode; private String errMessage; - public HttpEndPointResponse(final String url, + public HttpEndpointResponse(final String url, final int statusCode, final String errMessage) { this.url = url; @@ -17,7 +23,7 @@ public HttpEndPointResponse(final String url, this.errMessage = errMessage; } - public HttpEndPointResponse(final String url, + public HttpEndpointResponse(final String url, final int statusCode) { this.url = url; this.statusCode = statusCode; @@ -43,4 +49,4 @@ public String toString() { ", errMessage='" + errMessage + '\'' + '}'; } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpSink.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpSink.java new file mode 100644 index 0000000000..a7f5e8bee3 --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpSink.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http; + +import com.google.common.annotations.VisibleForTesting; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.common.sink.DefaultSinkMetrics; +import org.opensearch.dataprepper.common.sink.SinkMetrics; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.codec.OutputCodec; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.pipeline.HeadlessPipeline; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.sink.AbstractSink; +import org.opensearch.dataprepper.model.sink.OutputCodecContext; +import org.opensearch.dataprepper.model.sink.Sink; +import org.opensearch.dataprepper.model.sink.SinkContext; +import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; +import org.opensearch.dataprepper.plugins.sink.http.service.HttpSinkService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; + +@DataPrepperPlugin(name = "http", pluginType = Sink.class, pluginConfigurationType = HttpSinkConfiguration.class) +public class HttpSink extends AbstractSink> { + + private static final Logger LOG = LoggerFactory.getLogger(HttpSink.class); + + private volatile boolean sinkInitialized; + + private final HttpSinkService httpSinkService; + + @DataPrepperPluginConstructor + public HttpSink(final PluginSetting pluginSetting, + final HttpSinkConfiguration httpSinkConfiguration, + final PluginFactory pluginFactory, + final PipelineDescription pipelineDescription, + final SinkContext sinkContext, + final AwsCredentialsSupplier awsCredentialsSupplier, + final PluginMetrics pluginMetrics) { + super(pluginSetting); + this.sinkInitialized = false; + + final SinkContext context = sinkContext != null ? sinkContext : new SinkContext(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + final PluginModel codecConfiguration = httpSinkConfiguration.getCodec(); + final PluginSetting codecPluginSettings = new PluginSetting(codecConfiguration.getPluginName(), + codecConfiguration.getPluginSettings()); + codecPluginSettings.setPipelineName(pipelineDescription.getPipelineName()); + + final OutputCodec codec = pluginFactory.loadPlugin(OutputCodec.class, codecPluginSettings); + final OutputCodecContext codecContext = OutputCodecContext.fromSinkContext(context); + + final SinkMetrics sinkMetrics = new DefaultSinkMetrics(pluginMetrics, "Event"); + + final AuthenticationDecorator authDecorator = httpSinkConfiguration.getAwsConfig() != null + ? new AwsAuthenticationDecorator(awsCredentialsSupplier, httpSinkConfiguration.getAwsConfig(), + httpSinkConfiguration.getAwsSigv4ServiceName()) + : null; + + final HttpSinkSender httpSender = new HttpSinkSender(authDecorator, httpSinkConfiguration, sinkMetrics); + + this.httpSinkService = new HttpSinkService( + httpSinkConfiguration, + sinkMetrics, + httpSender, + pipelineDescription, + codec, + codecContext); + } + + @Override + public boolean isReady() { + return sinkInitialized; + } + + @Override + public void doInitialize() { + sinkInitialized = true; + httpSinkService.setDlqPipeline(getFailurePipeline()); + } + + @VisibleForTesting + void setDlqPipeline(HeadlessPipeline dlqPipeline) { httpSinkService.setDlqPipeline(dlqPipeline); } + + @Override + public void doOutput(final Collection> records) { + httpSinkService.output(records); + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkSender.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkSender.java new file mode 100644 index 0000000000..a50d6c918a --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkSender.java @@ -0,0 +1,151 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http; + +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.ClientOptions; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.RequestHeadersBuilder; +import org.opensearch.dataprepper.common.sink.SinkMetrics; +import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.net.URI; +import java.util.Set; + +public class HttpSinkSender { + private static final Logger LOG = LoggerFactory.getLogger(HttpSinkSender.class); + private static final Set RETRYABLE_STATUS_CODES = Set.of(408, 429, 500, 502, 503, 504); + private static final Set AUTH_ERROR_CODES = Set.of(401, 403); + + private final AuthenticationDecorator authenticationDecorator; + private final WebClient webClient; + private final HttpSinkConfiguration config; + private final int maxRetries; + private final long retryIntervalMs; + private final SinkMetrics sinkMetrics; + + public HttpSinkSender(final AuthenticationDecorator authenticationDecorator, + @Nonnull final HttpSinkConfiguration config, + final SinkMetrics sinkMetrics) { + this.authenticationDecorator = authenticationDecorator; + this.webClient = buildWebClient(config); + this.config = config; + this.sinkMetrics = sinkMetrics; + this.maxRetries = config.getMaxUploadRetries(); + this.retryIntervalMs = config.getHttpRetryInterval().toMillis(); + } + + private static WebClient buildWebClient(final HttpSinkConfiguration config) { + return WebClient.builder() + .factory(ClientFactory.builder() + .connectTimeout(config.getConnectionTimeout()) + .build()) + .options(ClientOptions.builder().build()) + .build(); + } + + public HttpEndpointResponse send(final byte[] payload) { + HttpEndpointResponse response = null; + int attempt = 0; + + while (attempt <= maxRetries) { + try { + final HttpRequest request = buildHttpRequest(payload); + + response = webClient.execute(request) + .aggregate() + .thenApply(resp -> { + int statusCode = resp.status().code(); + String responseBody = resp.content().toStringUtf8(); + + return new HttpEndpointResponse(config.getUrl(), statusCode, responseBody); + }) + .exceptionally(throwable -> { + LOG.error("Request failed", throwable); + return new HttpEndpointResponse(config.getUrl(), 0, throwable.getMessage()); + }) + .join(); + + if (response.getStatusCode() >= 200 && response.getStatusCode() < 300) { + return response; + } + + if (AUTH_ERROR_CODES.contains(response.getStatusCode())) { + LOG.error("Authentication error ({}), not retrying", response.getStatusCode()); + return response; + } + + if (!RETRYABLE_STATUS_CODES.contains(response.getStatusCode())) { + LOG.error("Non-retryable error ({}), message({}) not retrying", response.getStatusCode(), response.getErrMessage()); + return response; + } + + if (attempt < maxRetries) { + LOG.warn("Retryable error ({}), attempt {}/{}, retrying after {}ms", + response.getStatusCode(), attempt + 1, maxRetries, retryIntervalMs); + Thread.sleep(retryIntervalMs); + sinkMetrics.incrementRetries(1); + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOG.error("Retry interrupted", e); + return new HttpEndpointResponse(config.getUrl(), 0, "Retry interrupted: " + e.getMessage()); + } catch (Exception e) { + LOG.error("Failed to execute request, attempt {}/{}", attempt + 1, maxRetries + 1, e); + if (attempt >= maxRetries) { + return new HttpEndpointResponse(config.getUrl(), 0, e.getMessage()); + } + try { + Thread.sleep(retryIntervalMs); + sinkMetrics.incrementRetries(1); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return new HttpEndpointResponse(config.getUrl(), 0, "Retry interrupted: " + ie.getMessage()); + } + } + attempt++; + } + + return response != null ? response : new HttpEndpointResponse(config.getUrl(), 0, "Max retries exceeded"); + } + + private HttpRequest buildHttpRequest(final byte[] payload) { + if (authenticationDecorator != null) { + return authenticationDecorator.buildRequest(config.getUrl(), payload, config.getCustomHeaderOptions()); + } + return buildPlainHttpRequest(payload); + } + + private HttpRequest buildPlainHttpRequest(final byte[] payload) { + final URI uri = URI.create(config.getUrl()); + final RequestHeadersBuilder headersBuilder = RequestHeaders.builder() + .method(HttpMethod.POST) + .scheme(uri.getScheme()) + .path(uri.getRawPath()) + .authority(uri.getAuthority()); + + if (config.getCustomHeaderOptions() != null) { + config.getCustomHeaderOptions().forEach((key, values) -> + values.forEach(value -> headersBuilder.add(key, value)) + ); + } + + return HttpRequest.of(headersBuilder.build(), HttpData.wrap(payload)); + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/OAuthAccessTokenManager.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/OAuthAccessTokenManager.java deleted file mode 100644 index 9a7d1b2bae..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/OAuthAccessTokenManager.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http; - -import com.github.scribejava.core.builder.api.DefaultApi20; -import com.github.scribejava.core.model.OAuth2AccessToken; -import com.github.scribejava.core.oauth.OAuth20Service; -import com.github.scribejava.core.builder.ServiceBuilder; -import org.opensearch.dataprepper.plugins.sink.http.configuration.BearerTokenOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Instant; -import java.time.ZoneOffset; - -public class OAuthAccessTokenManager { - - private static final Logger LOG = LoggerFactory.getLogger(OAuthAccessTokenManager.class); - - public String getAccessToken(final BearerTokenOptions bearerTokenOptions) { - OAuth20Service service = getOAuth20ServiceObj(bearerTokenOptions); - OAuth2AccessToken accessTokenObj = null; - try { - if(bearerTokenOptions.getRefreshToken() != null) { - accessTokenObj = new OAuth2AccessToken(bearerTokenOptions.getAccessToken(), bearerTokenOptions.getRefreshToken()); - accessTokenObj = service.refreshAccessToken(accessTokenObj.getRefreshToken()); - - }else { - accessTokenObj = service.getAccessTokenClientCredentialsGrant(); - } - bearerTokenOptions.setRefreshToken(accessTokenObj.getRefreshToken()); - bearerTokenOptions.setAccessToken(accessTokenObj.getAccessToken()); - bearerTokenOptions.setTokenExpired(accessTokenObj.getExpiresIn()); - }catch (Exception e) { - LOG.info("Exception : "+ e.getMessage() ); - } - return bearerTokenOptions.getAccessToken(); - } - - - public boolean isTokenExpired(final Integer tokenExpired){ - final Instant systemCurrentTimeStamp = Instant.now().atOffset(ZoneOffset.UTC).toInstant(); - Instant accessTokenExpTimeStamp = systemCurrentTimeStamp.plusSeconds(tokenExpired); - if(systemCurrentTimeStamp.compareTo(accessTokenExpTimeStamp)>=0) { - return true; - } - return false; - } - - private OAuth20Service getOAuth20ServiceObj(BearerTokenOptions bearerTokenOptions){ - return new ServiceBuilder(bearerTokenOptions.getClientId()) - .apiSecret(bearerTokenOptions.getClientSecret()) - .defaultScope(bearerTokenOptions.getScope()) - .build(new DefaultApi20() { - @Override - public String getAccessTokenEndpoint() { - return bearerTokenOptions.getTokenUrl(); - } - - @Override - protected String getAuthorizationBaseUrl() { - return bearerTokenOptions.getTokenUrl(); - } - }); - } - -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/certificate/CertificateProviderFactory.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/certificate/CertificateProviderFactory.java deleted file mode 100644 index 332382e03c..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/certificate/CertificateProviderFactory.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.certificate; - -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.acm.ACMCertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.file.FileCertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.s3.S3CertificateProvider; -import org.opensearch.dataprepper.plugins.metricpublisher.MicrometerMetricPublisher; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.retry.RetryMode; -import software.amazon.awssdk.services.acm.AcmClient; -import software.amazon.awssdk.services.s3.S3Client; - -/** - * This class consist logic for downloading the SSL certificates from S3/ACM/Local file. - * - */ -public class CertificateProviderFactory { - private static final Logger LOG = LoggerFactory.getLogger(CertificateProviderFactory.class); - - final HttpSinkConfiguration httpSinkConfiguration; - public CertificateProviderFactory(final HttpSinkConfiguration httpSinkConfiguration) { - this.httpSinkConfiguration = httpSinkConfiguration; - } - - /** - * This method consist logic for downloading the SSL certificates from S3/ACM/Local file. - * @return CertificateProvider - */ - public CertificateProvider getCertificateProvider() { - if (httpSinkConfiguration.useAcmCertForSSL()) { - LOG.info("Using ACM certificate and private key for SSL/TLS."); - final AwsCredentialsProvider credentialsProvider = AwsCredentialsProviderChain.builder() - .addCredentialsProvider(DefaultCredentialsProvider.create()).build(); - final ClientOverrideConfiguration clientConfig = ClientOverrideConfiguration.builder() - .retryPolicy(RetryMode.STANDARD) - .build(); - - final PluginMetrics awsSdkMetrics = PluginMetrics.fromNames("sdk", "aws"); - - final AcmClient awsCertificateManager = AcmClient.builder() - .region(httpSinkConfiguration.getAwsAuthenticationOptions().getAwsRegion()) - .credentialsProvider(credentialsProvider) - .overrideConfiguration(clientConfig) - .overrideConfiguration(metricPublisher -> metricPublisher.addMetricPublisher(new MicrometerMetricPublisher(awsSdkMetrics))) - .build(); - - return new ACMCertificateProvider(awsCertificateManager, httpSinkConfiguration.getAcmCertificateArn(), - httpSinkConfiguration.getAcmCertIssueTimeOutMillis(), httpSinkConfiguration.getAcmPrivateKeyPassword()); - } else if (httpSinkConfiguration.isSslCertAndKeyFileInS3()) { - LOG.info("Using S3 to fetch certificate and private key for SSL/TLS."); - final AwsCredentialsProvider credentialsProvider = AwsCredentialsProviderChain.builder() - .addCredentialsProvider(DefaultCredentialsProvider.create()).build(); - final S3Client s3Client = S3Client.builder() - .region(httpSinkConfiguration.getAwsAuthenticationOptions().getAwsRegion()) - .credentialsProvider(credentialsProvider) - .build(); - return new S3CertificateProvider(s3Client, - httpSinkConfiguration.getSslCertificateFile(), - httpSinkConfiguration.getSslKeyFile()); - } else { - LOG.info("Using local file system to get certificate and private key for SSL/TLS."); - return new FileCertificateProvider(httpSinkConfiguration.getSslCertificateFile(), httpSinkConfiguration.getSslKeyFile()); - } - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/certificate/HttpClientSSLConnectionManager.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/certificate/HttpClientSSLConnectionManager.java deleted file mode 100644 index c33a4d3f99..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/certificate/HttpClientSSLConnectionManager.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.certificate; - -import org.apache.hc.client5.http.config.TlsConfig; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; -import org.apache.hc.client5.http.io.HttpClientConnectionManager; -import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; -import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; -import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder; -import org.apache.hc.client5.http.ssl.TrustAllStrategy; -import org.apache.hc.core5.http.ssl.TLS; -import org.apache.hc.core5.ssl.SSLContextBuilder; -import org.apache.hc.core5.ssl.SSLContexts; -import org.apache.hc.core5.ssl.TrustStrategy; -import org.apache.hc.core5.util.Timeout; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLContext; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.Certificate; -import java.security.cert.CertificateFactory; - -/** - * This class implements SSL certs authentication - * - */ -public class HttpClientSSLConnectionManager { - - private static final Logger LOG = LoggerFactory.getLogger(HttpClientSSLConnectionManager.class); - - /** - * This method creates HttpClientConnectionManager for SSL certs authentication - * @param sinkConfiguration HttpSinkConfiguration - * @param providerFactory CertificateProviderFactory - * @return HttpClientConnectionManager - */ - public HttpClientConnectionManager createHttpClientConnectionManager(final HttpSinkConfiguration sinkConfiguration, - final CertificateProviderFactory providerFactory){ - final SSLContext sslContext = sinkConfiguration.getSslCertificateFile() != null ? - getCAStrategy(new ByteArrayInputStream(providerFactory.getCertificateProvider().getCertificate().getCertificate().getBytes(StandardCharsets.UTF_8))) : getTrustAllStrategy(); - SSLConnectionSocketFactory sslSocketFactory = SSLConnectionSocketFactoryBuilder.create() - .setSslContext(sslContext) - .build(); - return PoolingHttpClientConnectionManagerBuilder.create() - .setSSLSocketFactory(sslSocketFactory) - .setDefaultTlsConfig(TlsConfig.custom() - .setHandshakeTimeout(Timeout.ofSeconds(30)) - .setSupportedProtocols(TLS.V_1_3) - .build()) - .build(); - } - - private SSLContext getCAStrategy(final InputStream certificate) { - try { - CertificateFactory factory = CertificateFactory.getInstance("X.509"); - Certificate trustedCa; - trustedCa = factory.generateCertificate(certificate); - KeyStore trustStore = KeyStore.getInstance("pkcs12"); - trustStore.load(null, null); - trustStore.setCertificateEntry("ca", trustedCa); - SSLContextBuilder sslContextBuilder = SSLContexts.custom() - .loadTrustMaterial(trustStore, null); - return sslContextBuilder.build(); - } catch (Exception ex) { - throw new RuntimeException(ex.getMessage(), ex); - } - } - - private SSLContext getTrustAllStrategy() { - final TrustStrategy trustStrategy = new TrustAllStrategy(); - try { - return SSLContexts.custom().loadTrustMaterial(null, trustStrategy).build(); - } catch (Exception ex) { - throw new RuntimeException(ex.getMessage(), ex); - } - } - - public HttpClientConnectionManager createHttpClientConnectionManagerWithoutValidation() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - { - return PoolingHttpClientConnectionManagerBuilder.create() - .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create() - .setSslContext(SSLContextBuilder.create() - .loadTrustMaterial(TrustAllStrategy.INSTANCE) - .build()) - .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .build()) - .build(); - } - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AuthTypeOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AuthTypeOptions.java deleted file mode 100644 index 88d9b6cfe8..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AuthTypeOptions.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.annotation.JsonCreator; - -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Collectors; - -public enum AuthTypeOptions { - HTTP_BASIC("http_basic"), - BEARER_TOKEN("bearer_token"), - UNAUTHENTICATED("unauthenticated"); - - private static final Map OPTIONS_MAP = Arrays.stream(AuthTypeOptions.values()) - .collect(Collectors.toMap( - value -> value.option, - value -> value - )); - - private final String option; - - AuthTypeOptions(final String option) { - this.option = option; - } - - @JsonCreator - static AuthTypeOptions fromOptionValue(final String option) { - return OPTIONS_MAP.get(option); - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AuthenticationOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AuthenticationOptions.java deleted file mode 100644 index 1622967696..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AuthenticationOptions.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class AuthenticationOptions { - - @JsonProperty("http_basic") - private BasicAuthCredentials httpBasic; - - @JsonProperty("bearer_token") - private BearerTokenOptions bearerTokenOptions; - - public BasicAuthCredentials getHttpBasic() { - return httpBasic; - } - - public BearerTokenOptions getBearerTokenOptions() { - return bearerTokenOptions; - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AwsAuthenticationOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AwsAuthenticationOptions.java deleted file mode 100644 index 703c635fa2..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AwsAuthenticationOptions.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.Size; -import software.amazon.awssdk.regions.Region; - -import java.util.Map; - -public class AwsAuthenticationOptions { - - private static final String DEFAULT_SERVICE_NAME = "execute-api"; - @JsonProperty("region") - @Size(min = 1, message = "Region cannot be empty string") - private String awsRegion; - - @JsonProperty("sts_role_arn") - @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") - private String awsStsRoleArn; - - @JsonProperty("sts_external_id") - @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") - private String awsStsExternalId; - - @JsonProperty("sts_header_overrides") - @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") - private Map awsStsHeaderOverrides; - - @JsonProperty("service_name") - @Size(min = 1, message = "Service Name cannot be empty") - private String serviceName = DEFAULT_SERVICE_NAME; - - public Region getAwsRegion() { - return awsRegion != null ? Region.of(awsRegion) : null; - } - - public String getAwsStsRoleArn() { - return awsStsRoleArn; - } - - public String getAwsStsExternalId() { - return awsStsExternalId; - } - - public Map getAwsStsHeaderOverrides() { - return awsStsHeaderOverrides; - } - - public String getServiceName() { - return serviceName; - } -} \ No newline at end of file diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/BasicAuthCredentials.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/BasicAuthCredentials.java deleted file mode 100644 index 9f5defd1ef..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/BasicAuthCredentials.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class BasicAuthCredentials { - - - @JsonProperty("username") - private String username; - - @JsonProperty("password") - private String password; - - public String getUsername() { - return username; - } - - public String getPassword() { - return password; - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/BearerTokenOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/BearerTokenOptions.java deleted file mode 100644 index 89ec7dd834..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/BearerTokenOptions.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.NotNull; - -public class BearerTokenOptions { - - @JsonProperty("client_id") - @NotNull(message = "client id type is mandatory for refresh token") - private String clientId; - - @JsonProperty("client_secret") - @NotNull(message = "client secret type is mandatory for refresh token") - private String clientSecret; - - @JsonProperty("token_url") - @NotNull(message = "token url type is mandatory for refresh token") - private String tokenUrl; - - @JsonProperty("grant_type") - @NotNull(message = "grant type is mandatory for refresh token") - private String grantType; - - @JsonProperty("scope") - @NotNull(message = "scope is mandatory for refresh token") - private String scope; - - private String refreshToken; - - private String accessToken; - - private Integer tokenExpired; - - public String getScope() { - return scope; - } - - public String getGrantType() { - return grantType; - } - - public String getRefreshToken() { - return refreshToken; - } - - public String getAccessToken() { - return accessToken; - } - - public Integer getTokenExpired() { - return tokenExpired; - } - - public String getClientId() { - return clientId; - } - - public String getClientSecret() { - return clientSecret; - } - - public void setRefreshToken(String refreshToken) { - this.refreshToken = refreshToken; - } - - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } - - public void setTokenExpired(Integer tokenExpired) { - this.tokenExpired = tokenExpired; - } - - public String getTokenUrl() { - return tokenUrl; - } - -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HTTPMethodOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HTTPMethodOptions.java deleted file mode 100644 index 85749f9819..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HTTPMethodOptions.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.annotation.JsonCreator; - -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Collectors; - -public enum HTTPMethodOptions { - PUT("PUT"), - POST("POST"); - - private static final Map OPTIONS_MAP = Arrays.stream(HTTPMethodOptions.values()) - .collect(Collectors.toMap( - value -> value.option, - value -> value - )); - - private final String option; - - HTTPMethodOptions(final String option) { - this.option = option; - } - - @JsonCreator - static HTTPMethodOptions fromOptionValue(final String option) { - return OPTIONS_MAP.get(option); - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfiguration.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfiguration.java index e1e9ac9c57..22bfd2ba7a 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfiguration.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfiguration.java @@ -1,93 +1,38 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.configuration; import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; -import org.apache.commons.lang3.StringUtils; import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.plugins.accumulator.BufferTypeOptions; -import org.opensearch.dataprepper.plugins.sink.http.util.HttpSinkUtil; +import org.opensearch.dataprepper.aws.api.AwsConfig; -import java.net.URL; import java.time.Duration; -import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.List; public class HttpSinkConfiguration { - - private static final int DEFAULT_UPLOAD_RETRIES = 5; - - private static final int DEFAULT_WORKERS = 1; - static final boolean DEFAULT_INSECURE = false; - - private static final String S3_PREFIX = "s3://"; - - static final String SSL_KEY_CERT_FILE = "sslKeyCertChainFile"; - static final String SSL_KEY_FILE = "sslKeyFile"; - static final String SSL = "ssl"; - static final String AWS_REGION = "awsRegion"; - - - public static final String STS_REGION = "sts_region"; - - public static final String STS_ROLE_ARN = "sts_role_arn"; - static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; - static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000; - public static final String SSL_IS_ENABLED = "%s is enabled"; - + static final int DEFAULT_UPLOAD_RETRIES = 3; public static final Duration DEFAULT_HTTP_RETRY_INTERVAL = Duration.ofSeconds(30); - - private static final String HTTPS = "https"; - - private static final String HTTP = "http"; - - private static final String AWS_HOST_AMAZONAWS_COM = "amazonaws.com"; - - private static final String AWS_HOST_API_AWS = "api.aws"; - - private static final String AWS_HOST_ON_AWS = "on.aws"; + public static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(60); @NotNull @JsonProperty("url") private String url; - @JsonProperty("workers") - private Integer workers = DEFAULT_WORKERS; - @JsonProperty("codec") private PluginModel codec; - @JsonProperty("http_method") - private HTTPMethodOptions httpMethod = HTTPMethodOptions.POST; - - @JsonProperty("proxy") - private String proxy; - - @JsonProperty("auth_type") - private AuthTypeOptions authType = AuthTypeOptions.UNAUTHENTICATED; - - @JsonProperty("authentication") - private AuthenticationOptions authentication; - - @JsonProperty("ssl_certificate_file") - private String sslCertificateFile; - - @JsonProperty("ssl_key_file") - private String sslKeyFile; - - @JsonProperty("aws_sigv4") - private boolean awsSigv4; - - @JsonProperty("buffer_type") - private BufferTypeOptions bufferType = BufferTypeOptions.INMEMORY; - - @NotNull @JsonProperty("threshold") private ThresholdOptions thresholdOptions; @@ -96,31 +41,13 @@ public class HttpSinkConfiguration { @JsonProperty("aws") @Valid - private AwsAuthenticationOptions awsAuthenticationOptions; - - @JsonProperty("custom_header") - private Map> customHeaderOptions; - - @JsonProperty("dlq_file") - private String dlqFile; - - @JsonProperty("webhook_url") - private String webhookURL; - - @JsonProperty("dlq") - private PluginModel dlq; - - @JsonProperty("use_acm_cert_for_ssl") - private boolean useAcmCertForSSL = DEFAULT_USE_ACM_CERT_FOR_SSL; + private AwsConfig awsConfig; - @JsonProperty("acm_private_key_password") - private String acmPrivateKeyPassword; - - @JsonProperty("acm_certificate_arn") - private String acmCertificateArn; + @JsonProperty("http_retry_interval") + private Duration httpRetryInterval = DEFAULT_HTTP_RETRY_INTERVAL; - @JsonProperty("acm_cert_issue_time_out_millis") - private long acmCertIssueTimeOutMillis = DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS; + @JsonProperty("request_timeout") + private Duration requestTimeout; @JsonProperty("insecure") private boolean insecure = DEFAULT_INSECURE; @@ -128,179 +55,61 @@ public class HttpSinkConfiguration { @JsonProperty("insecure_skip_verify") private boolean insecureSkipVerify = DEFAULT_INSECURE; - @JsonProperty("request_timout") - private Duration requestTimout; - - @JsonProperty("http_retry_interval") - private Duration httpRetryInterval = DEFAULT_HTTP_RETRY_INTERVAL; + @JsonProperty("connection_timeout") + private Duration connectionTimeout = DEFAULT_CONNECTION_TIMEOUT; + @JsonProperty("aws_sigv4_service_name") + private String awsSigv4ServiceName; - private boolean sslCertAndKeyFileInS3; + @JsonProperty("custom_headers") + private Map> customHeaderOptions; public String getUrl() { return url; } - public boolean isInsecureSkipVerify() { - return insecureSkipVerify; - } - - public Duration getHttpRetryInterval() { - return httpRetryInterval; - } - - public String getAcmPrivateKeyPassword() { - return acmPrivateKeyPassword; - } - - public boolean isSslCertAndKeyFileInS3() { - return sslCertAndKeyFileInS3; - } - - public long getAcmCertIssueTimeOutMillis() { - return acmCertIssueTimeOutMillis; - } - - public boolean useAcmCertForSSL() { - return useAcmCertForSSL; - } - - public String getWebhookURL() { - return webhookURL; - } - - public void validateAndInitializeCertAndKeyFileInS3() { - boolean certAndKeyFileInS3 = false; - if (useAcmCertForSSL) { - validateSSLArgument(String.format(SSL_IS_ENABLED, useAcmCertForSSL), acmCertificateArn, acmCertificateArn); - validateSSLArgument(String.format(SSL_IS_ENABLED, useAcmCertForSSL), awsAuthenticationOptions.getAwsRegion().toString(), AWS_REGION); - } else if(!insecureSkipVerify) { - validateSSLCertificateFiles(); - certAndKeyFileInS3 = isSSLCertificateLocatedInS3(); - if (certAndKeyFileInS3) { - validateSSLArgument("The certificate and key files are located in S3", awsAuthenticationOptions.getAwsRegion().toString(), AWS_REGION); - } - } - sslCertAndKeyFileInS3 = certAndKeyFileInS3; - } - private void validateSSLArgument(final String sslTypeMessage, final String argument, final String argumentName) { - if (StringUtils.isEmpty(argument)) { - throw new IllegalArgumentException(String.format("%s, %s can not be empty or null", sslTypeMessage, argumentName)); - } - } - - private void validateSSLCertificateFiles() { - validateSSLArgument(String.format(SSL_IS_ENABLED, SSL), sslCertificateFile, SSL_KEY_CERT_FILE); - validateSSLArgument(String.format(SSL_IS_ENABLED, SSL), sslKeyFile, SSL_KEY_FILE); - } - - private boolean isSSLCertificateLocatedInS3() { - return sslCertificateFile.toLowerCase().startsWith(S3_PREFIX) && - sslKeyFile.toLowerCase().startsWith(S3_PREFIX); - } - - public String getAcmCertificateArn() { - return acmCertificateArn; - } - public PluginModel getCodec() { return codec; } - public HTTPMethodOptions getHttpMethod() { - return httpMethod; - } - - public String getProxy() { - return proxy; - } - - public AuthTypeOptions getAuthType() { - return authType; - } - - public AuthenticationOptions getAuthentication() { - return authentication; - } - - public String getSslCertificateFile() { - return sslCertificateFile; - } - - public String getSslKeyFile() { - return sslKeyFile; - } - - public boolean isAwsSigv4() { - return awsSigv4; - } - - public BufferTypeOptions getBufferType() { - return bufferType; - } - public ThresholdOptions getThresholdOptions() { + if (thresholdOptions == null) { + return new ThresholdOptions(); + } return thresholdOptions; } - public int getMaxUploadRetries() { - return maxUploadRetries; - } - public Map> getCustomHeaderOptions() { return customHeaderOptions; } - public AwsAuthenticationOptions getAwsAuthenticationOptions() { - return awsAuthenticationOptions; - } - - public Integer getWorkers() { - return workers; - } - - public String getDlqFile() { - return dlqFile; - } - - public PluginModel getDlq() { - return dlq; + public int getMaxUploadRetries() { + return maxUploadRetries; } - public boolean isValidAWSUrl() { - URL parsedUrl = HttpSinkUtil.getURLByUrlString(url); - if(parsedUrl.getProtocol().equals(HTTPS) && (parsedUrl.getHost().contains(AWS_HOST_AMAZONAWS_COM) ||parsedUrl.getHost().contains(AWS_HOST_API_AWS) || parsedUrl.getHost().contains(AWS_HOST_ON_AWS))){ - return true; - } - return false; + public AwsConfig getAwsConfig() { + return awsConfig; } - public String getDlqStsRoleARN(){ - return Objects.nonNull(getDlqPluginSetting().get(STS_ROLE_ARN)) ? - String.valueOf(getDlqPluginSetting().get(STS_ROLE_ARN)) : - awsAuthenticationOptions.getAwsStsRoleArn(); + public Duration getHttpRetryInterval() { + return httpRetryInterval; } - public String getDlqStsRegion(){ - return Objects.nonNull(getDlqPluginSetting().get(STS_REGION)) ? - String.valueOf(getDlqPluginSetting().get(STS_REGION)) : - awsAuthenticationOptions.getAwsRegion().toString(); + public Duration getRequestTimeout() { + return requestTimeout; } - public Map getDlqPluginSetting(){ - return dlq != null ? dlq.getPluginSettings() : Map.of(); + public String getAwsSigv4ServiceName() { + return awsSigv4ServiceName; } - public boolean isInsecure() { - return insecure; + public Duration getConnectionTimeout() { + return connectionTimeout; } - public Duration getRequestTimout() { - return requestTimout; + @Valid + public boolean isValidConfig() { + return (insecure && awsConfig == null) || (!insecure && awsConfig != null); } - public boolean isHttpUrl() { - URL parsedUrl = HttpSinkUtil.getURLByUrlString(url); - return parsedUrl.getProtocol().equals(HTTP); - } } diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptions.java index dc8f904738..17ef37ab77 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptions.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptions.java @@ -1,7 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.configuration; import com.fasterxml.jackson.annotation.JsonProperty; @@ -18,43 +24,45 @@ */ public class ThresholdOptions { - private static final String DEFAULT_BYTE_CAPACITY = "50mb"; + private static final String DEFAULT_MAX_REQUEST_SIZE = "50mb"; + private static final int DEFAULT_MAX_EVENTS = 100; + private static final Duration DEFAULT_FLUSH_TIMEOUT = Duration.ofSeconds(10); - @JsonProperty("event_count") - @Size(min = 0, max = 10000000, message = "event_count size should be between 0 and 10000000") + @JsonProperty("max_events") + @Size(min = 1, max = 10000000, message = "event_count size should be between 0 and 10000000") @NotNull - private int eventCount; + private int maxEvents = DEFAULT_MAX_EVENTS; - @JsonProperty("maximum_size") - private String maximumSize = DEFAULT_BYTE_CAPACITY; + @JsonProperty("max_request_size") + private String maxRequestSize = DEFAULT_MAX_REQUEST_SIZE; - @JsonProperty("event_collect_timeout") + @JsonProperty("flush_timeout") @DurationMin(seconds = 1) @DurationMax(seconds = 3600) @NotNull - private Duration eventCollectTimeOut; + private Duration flushTimeout = DEFAULT_FLUSH_TIMEOUT; /** * Read event collection duration configuration. * @return event collect time out. */ - public Duration getEventCollectTimeOut() { - return eventCollectTimeOut; + public Duration getFlushTimeOut() { + return flushTimeout; } /** * Read byte capacity configuration. * @return maximum byte count. */ - public ByteCount getMaximumSize() { - return ByteCount.parse(maximumSize); + public ByteCount getMaxRequestSize() { + return ByteCount.parse(maxRequestSize); } /** * Read the event count configuration. * @return event count. */ - public int getEventCount() { - return eventCount; + public int getMaxEvents() { + return maxEvents; } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/dlq/DlqPushHandler.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/dlq/DlqPushHandler.java deleted file mode 100644 index 9c00561e8e..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/dlq/DlqPushHandler.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.dlq; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import io.micrometer.core.instrument.util.StringUtils; -import org.opensearch.dataprepper.metrics.MetricNames; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.failures.DlqObject; -import org.opensearch.dataprepper.model.plugin.PluginFactory; -import org.opensearch.dataprepper.plugins.dlq.DlqProvider; -import org.opensearch.dataprepper.plugins.dlq.DlqWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.StringJoiner; - -import static java.util.UUID.randomUUID; - - -/** - * * An Handler class which helps log failed data to AWS S3 bucket or file based on configuration. - */ - -public class DlqPushHandler { - - private static final Logger LOG = LoggerFactory.getLogger(DlqPushHandler.class); - - private static final String BUCKET = "bucket"; - - private static final String ROLE_ARN = "sts_role_arn"; - - private static final String REGION = "region"; - - private static final String S3_PLUGIN_NAME = "s3"; - - private static final String KEY_PATH_PREFIX = "key_path_prefix"; - - private String dlqFile; - - private String keyPathPrefix; - - private DlqProvider dlqProvider; - - private ObjectWriter objectWriter; - - public DlqPushHandler(final String dlqFile, - final PluginFactory pluginFactory, - final String bucket, - final String stsRoleArn, - final String awsRegion, - final String dlqPathPrefix) { - if(dlqFile != null) { - this.dlqFile = dlqFile; - this.objectWriter = new ObjectMapper().writer(); - }else{ - this.dlqProvider = getDlqProvider(pluginFactory,bucket,stsRoleArn,awsRegion,dlqPathPrefix); - } - } - - public void perform(final PluginSetting pluginSetting, - final Object failedData) { - if(dlqFile != null) - writeToFile(failedData); - else - pushToS3(pluginSetting, failedData); - } - - private void writeToFile(Object failedData) { - try(BufferedWriter dlqFileWriter = Files.newBufferedWriter(Paths.get(dlqFile), - StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { - dlqFileWriter.write(objectWriter.writeValueAsString(failedData)+"\n"); - - } catch (IOException e) { - LOG.error("Exception while writing failed data to DLQ file Exception: ",e); - } - } - - private void pushToS3(PluginSetting pluginSetting, Object failedData) { - DlqWriter dlqWriter = getDlqWriter(pluginSetting.getPipelineName()); - try { - String pluginId = randomUUID().toString(); - DlqObject dlqObject = DlqObject.builder() - .withPluginId(pluginId) - .withPluginName(pluginSetting.getName()) - .withPipelineName(pluginSetting.getPipelineName()) - .withFailedData(failedData) - .build(); - - dlqWriter.write(Arrays.asList(dlqObject), pluginSetting.getPipelineName(), pluginId); - } catch (final IOException e) { - LOG.error("Exception while writing failed data to DLQ, Exception : ", e); - } - } - - private DlqWriter getDlqWriter(final String writerPipelineName) { - Optional potentialDlq = dlqProvider.getDlqWriter(new StringJoiner(MetricNames.DELIMITER) - .add(writerPipelineName).toString()); - DlqWriter dlqWriter = potentialDlq.isPresent() ? potentialDlq.get() : null; - return dlqWriter; - } - - private DlqProvider getDlqProvider(final PluginFactory pluginFactory, - final String bucket, - final String stsRoleArn, - final String awsRegion, - final String dlqPathPrefix) { - final Map props = new HashMap<>(); - props.put(BUCKET, bucket); - props.put(ROLE_ARN, stsRoleArn); - props.put(REGION, awsRegion); - this.keyPathPrefix = StringUtils.isEmpty(dlqPathPrefix) ? dlqPathPrefix : enforceDefaultDelimiterOnKeyPathPrefix(dlqPathPrefix); - props.put(KEY_PATH_PREFIX, dlqPathPrefix); - final PluginSetting dlqPluginSetting = new PluginSetting(S3_PLUGIN_NAME, props); - DlqProvider dlqProvider = pluginFactory.loadPlugin(DlqProvider.class, dlqPluginSetting); - return dlqProvider; - } - - private String enforceDefaultDelimiterOnKeyPathPrefix(final String keyPathPrefix) { - return (keyPathPrefix.charAt(keyPathPrefix.length() - 1) == '/') ? keyPathPrefix : keyPathPrefix.concat("/"); - } -} - diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/dlq/FailedDlqData.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/dlq/FailedDlqData.java deleted file mode 100644 index 024710ea8c..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/dlq/FailedDlqData.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.dlq; - -public class FailedDlqData { - - private String url; - - private int status; - - private String message; - - public FailedDlqData(final Builder builder) { - this.status = builder.status; - this.message = builder.message; - this.url = builder.url; - } - - public String getUrl() { - return url; - } - - public int getStatus() { - return status; - } - - public String getMessage() { - return message; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - - private String url; - - private int status; - - private String message; - - public Builder withUrl(String url) { - this.url = url; - return this; - } - - public Builder withStatus(int status) { - this.status = status; - return this; - } - - public Builder withMessage(String message) { - this.message = message; - return this; - } - - public FailedDlqData build() { - return new FailedDlqData(this); - } - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/BasicAuthHttpSinkHandler.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/BasicAuthHttpSinkHandler.java deleted file mode 100644 index 1078551d4d..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/BasicAuthHttpSinkHandler.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.handler; - -import org.apache.hc.client5.http.auth.AuthScope; -import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; -import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; -import org.apache.hc.client5.http.io.HttpClientConnectionManager; -import org.opensearch.dataprepper.plugins.sink.http.FailedHttpResponseInterceptor; -import org.opensearch.dataprepper.plugins.sink.http.util.HttpSinkUtil; - -/** - * * This class handles Basic Authentication - */ -public class BasicAuthHttpSinkHandler implements MultiAuthHttpSinkHandler { - - private final HttpClientConnectionManager httpClientConnectionManager; - - private final String username; - - private final String password; - - public BasicAuthHttpSinkHandler(final String username, - final String password, - final HttpClientConnectionManager httpClientConnectionManager){ - this.httpClientConnectionManager = httpClientConnectionManager; - this.username = username; - this.password = password; - } - - @Override - public HttpAuthOptions authenticate(final HttpAuthOptions.Builder httpAuthOptionsBuilder) { - final BasicCredentialsProvider provider = new BasicCredentialsProvider(); - AuthScope authScope = new AuthScope(HttpSinkUtil.getHttpHostByURL(HttpSinkUtil.getURLByUrlString(httpAuthOptionsBuilder.getUrl()))); - provider.setCredentials(authScope, new UsernamePasswordCredentials(username, password.toCharArray())); - httpAuthOptionsBuilder.setHttpClientBuilder(httpAuthOptionsBuilder.build().getHttpClientBuilder() - .setConnectionManager(httpClientConnectionManager) - .addResponseInterceptorLast(new FailedHttpResponseInterceptor(httpAuthOptionsBuilder.getUrl())) - .setDefaultCredentialsProvider(provider)); - return httpAuthOptionsBuilder.build(); - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/BearerTokenAuthHttpSinkHandler.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/BearerTokenAuthHttpSinkHandler.java deleted file mode 100644 index a48ffc7936..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/BearerTokenAuthHttpSinkHandler.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.handler; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.hc.client5.http.io.HttpClientConnectionManager; -import org.opensearch.dataprepper.plugins.sink.http.OAuthAccessTokenManager; -import org.opensearch.dataprepper.plugins.sink.http.configuration.BearerTokenOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.opensearch.dataprepper.plugins.sink.http.FailedHttpResponseInterceptor; - -/** - * * This class handles Bearer Token Authentication - */ -public class BearerTokenAuthHttpSinkHandler implements MultiAuthHttpSinkHandler { - - private static final Logger LOG = LoggerFactory.getLogger(BearerTokenAuthHttpSinkHandler.class); - - public static final String AUTHORIZATION = "Authorization"; - - private final HttpClientConnectionManager httpClientConnectionManager; - - private final BearerTokenOptions bearerTokenOptions; - - private final ObjectMapper objectMapper; - - private OAuthAccessTokenManager oAuthRefreshTokenManager; - - public BearerTokenAuthHttpSinkHandler(final BearerTokenOptions bearerTokenOptions, - final HttpClientConnectionManager httpClientConnectionManager, - final OAuthAccessTokenManager oAuthRefreshTokenManager){ - this.bearerTokenOptions = bearerTokenOptions; - this.httpClientConnectionManager = httpClientConnectionManager; - this.objectMapper = new ObjectMapper(); - this.oAuthRefreshTokenManager = oAuthRefreshTokenManager; - } - - @Override - public HttpAuthOptions authenticate(final HttpAuthOptions.Builder httpAuthOptionsBuilder) { - httpAuthOptionsBuilder.getClassicHttpRequestBuilder() - .addHeader(AUTHORIZATION, oAuthRefreshTokenManager.getAccessToken(bearerTokenOptions)); - httpAuthOptionsBuilder.setHttpClientBuilder(httpAuthOptionsBuilder.build().getHttpClientBuilder() - .setConnectionManager(httpClientConnectionManager) - .addResponseInterceptorLast(new FailedHttpResponseInterceptor(httpAuthOptionsBuilder.getUrl()))); - return httpAuthOptionsBuilder.build(); - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/HttpAuthOptions.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/HttpAuthOptions.java deleted file mode 100644 index 7270e60c48..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/HttpAuthOptions.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.handler; - -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.io.HttpClientConnectionManager; -import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; - - -public class HttpAuthOptions { - - private String url; - - private HttpClientBuilder httpClientBuilder; - - private ClassicRequestBuilder classicHttpRequestBuilder; - - private HttpClientConnectionManager httpClientConnectionManager; - - private int workers; - - private HttpHost proxy; - - public HttpClientBuilder getHttpClientBuilder() { - return httpClientBuilder; - } - - public ClassicRequestBuilder getClassicHttpRequestBuilder() { - return classicHttpRequestBuilder; - } - - public int getWorkers() { - return workers; - } - - public String getUrl() { - return url; - } - - public HttpHost getProxy() { - return proxy; - } - - public HttpClientConnectionManager getHttpClientConnectionManager() { - return httpClientConnectionManager; - } - - private HttpAuthOptions(Builder builder) { - this.url = builder.url; - this.httpClientBuilder = builder.httpClientBuilder; - this.classicHttpRequestBuilder = builder.classicHttpRequestBuilder; - this.httpClientConnectionManager = builder.httpClientConnectionManager; - this.workers = builder.workers; - this.proxy = builder.proxy; - } - public static class Builder { - - private String url; - private HttpClientBuilder httpClientBuilder; - private ClassicRequestBuilder classicHttpRequestBuilder; - private HttpClientConnectionManager httpClientConnectionManager; - private int workers; - private HttpHost proxy; - - public HttpAuthOptions build() { - return new HttpAuthOptions(this); - } - - public Builder setUrl(String url) { - this.url = url; - return this; - } - - public String getUrl() { - return url; - } - - public Builder setHttpClientBuilder(HttpClientBuilder httpClientBuilder) { - this.httpClientBuilder = httpClientBuilder; - return this; - } - - public Builder setClassicHttpRequestBuilder(ClassicRequestBuilder classicHttpRequestBuilder) { - this.classicHttpRequestBuilder = classicHttpRequestBuilder; - return this; - } - - public Builder setHttpClientConnectionManager(HttpClientConnectionManager httpClientConnectionManager) { - this.httpClientConnectionManager = httpClientConnectionManager; - return this; - } - - public Builder setWorkers(int workers) { - this.workers = workers; - return this; - } - - public Builder setProxy(HttpHost proxy) { - this.proxy = proxy; - return this; - } - - public ClassicRequestBuilder getClassicHttpRequestBuilder() { - return classicHttpRequestBuilder; - } - } - -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/MultiAuthHttpSinkHandler.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/MultiAuthHttpSinkHandler.java deleted file mode 100644 index ac295043b0..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/handler/MultiAuthHttpSinkHandler.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.handler; - -/** - * An interface to handle multiple authentications - */ -public interface MultiAuthHttpSinkHandler { - - /** - * This method can be used to implement multiple authentication based on configuration - * @param httpAuthOptionsBuilder HttpAuthOptions.Builder - * @return HttpAuthOptions - */ - HttpAuthOptions authenticate(final HttpAuthOptions.Builder httpAuthOptionsBuilder); - -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkAwsService.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkAwsService.java deleted file mode 100644 index 3b838b9c74..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkAwsService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.service; - -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; -import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.plugins.sink.http.AwsRequestSigningApacheInterceptor; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.signer.Aws4Signer; - -public class HttpSinkAwsService { - - private static final Logger LOG = LoggerFactory.getLogger(HttpSinkAwsService.class); - public static final String AWS_SIGV4 = "aws_sigv4"; - - public static void attachSigV4(final HttpSinkConfiguration httpSinkConfiguration, final HttpClientBuilder httpClientBuilder, final AwsCredentialsSupplier awsCredentialsSupplier) { - LOG.info("{} is set, will sign requests using AWSRequestSigningApacheInterceptor", AWS_SIGV4); - final Aws4Signer aws4Signer = Aws4Signer.create(); - final AwsCredentialsOptions awsCredentialsOptions = createAwsCredentialsOptions(httpSinkConfiguration); - final AwsCredentialsProvider credentialsProvider = awsCredentialsSupplier.getProvider(awsCredentialsOptions); - httpClientBuilder.addRequestInterceptorLast(new AwsRequestSigningApacheInterceptor(httpSinkConfiguration.getAwsAuthenticationOptions().getServiceName(), aws4Signer, - credentialsProvider, httpSinkConfiguration.getAwsAuthenticationOptions().getAwsRegion())); - } - - private static AwsCredentialsOptions createAwsCredentialsOptions(final HttpSinkConfiguration httpSinkConfiguration) { - return AwsCredentialsOptions.builder() - .withStsRoleArn(httpSinkConfiguration.getAwsAuthenticationOptions().getAwsStsRoleArn()) - .withStsExternalId(httpSinkConfiguration.getAwsAuthenticationOptions().getAwsStsExternalId()) - .withRegion(httpSinkConfiguration.getAwsAuthenticationOptions().getAwsRegion()) - .withStsHeaderOverrides(httpSinkConfiguration.getAwsAuthenticationOptions().getAwsStsHeaderOverrides()) - .build(); - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBuffer.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBuffer.java new file mode 100644 index 0000000000..63ced3566c --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBuffer.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http.service; + +import org.opensearch.dataprepper.common.sink.DefaultSinkBuffer; +import org.opensearch.dataprepper.common.sink.SinkBufferEntry; +import org.opensearch.dataprepper.common.sink.SinkBufferWriter; + +public class HttpSinkBuffer extends DefaultSinkBuffer { + + public HttpSinkBuffer(final long maxEvents, final long maxRequestSize, final long flushIntervalMs, final SinkBufferWriter sinkBufferWriter) { + super(maxEvents, maxRequestSize, flushIntervalMs, sinkBufferWriter); + } + + @Override + public boolean isMaxEventsLimitReached() { + return sinkBufferWriter.isMaxEventsLimitReached(maxEvents); + } + + @Override + public boolean willExceedMaxRequestSizeBytes(final SinkBufferEntry sinkBufferEntry) { + return sinkBufferWriter.willExceedMaxRequestSizeBytes(sinkBufferEntry, maxRequestSize); + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBufferEntry.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBufferEntry.java new file mode 100644 index 0000000000..b1c2d5b1d8 --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBufferEntry.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http.service; + +import org.opensearch.dataprepper.common.sink.SinkBufferEntry; +import org.opensearch.dataprepper.model.codec.OutputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.sink.OutputCodecContext; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class HttpSinkBufferEntry implements SinkBufferEntry { + private final Event event; + private final long estimatedSize; + + public HttpSinkBufferEntry(final Event event, final OutputCodec codec, final OutputCodecContext codecContext) throws IOException { + this.event = event; + this.estimatedSize = calculateSize(event, codec, codecContext); + } + + private long calculateSize(final Event event, final OutputCodec codec, final OutputCodecContext codecContext) throws IOException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + codec.start(outputStream, event, codecContext); + codec.writeEvent(event, outputStream); + codec.complete(outputStream); + return outputStream.size(); + } + } + + @Override + public long getEstimatedSize() { + return estimatedSize; + } + + @Override + public boolean exceedsMaxEventSizeThreshold() { + return false; + } + + @Override + public Event getEvent() { + return event; + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBufferWriter.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBufferWriter.java new file mode 100644 index 0000000000..5bfd9c1913 --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkBufferWriter.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http.service; + +import org.opensearch.dataprepper.common.sink.SinkBufferEntry; +import org.opensearch.dataprepper.common.sink.SinkBufferWriter; +import org.opensearch.dataprepper.common.sink.SinkFlushContext; +import org.opensearch.dataprepper.common.sink.SinkFlushableBuffer; +import org.opensearch.dataprepper.common.sink.SinkMetrics; + +import java.util.ArrayList; +import java.util.List; + +public class HttpSinkBufferWriter implements SinkBufferWriter { + private final List buffer; + private final SinkMetrics sinkMetrics; + private long currentSize; + + public HttpSinkBufferWriter(final SinkMetrics sinkMetrics) { + this.buffer = new ArrayList<>(); + this.sinkMetrics = sinkMetrics; + this.currentSize = 0; + } + + @Override + public boolean writeToBuffer(final SinkBufferEntry sinkBufferEntry) { + buffer.add(sinkBufferEntry); + currentSize += sinkBufferEntry.getEstimatedSize(); + return true; + } + + @Override + public SinkFlushableBuffer getBuffer(final SinkFlushContext sinkFlushContext) { + if (buffer.isEmpty()) { + return null; + } + final List bufferCopy = new ArrayList<>(buffer); + final long requestSize = currentSize; + buffer.clear(); + currentSize = 0; + sinkMetrics.recordRequestSize(requestSize); + return new HttpSinkFlushableBuffer(bufferCopy, sinkMetrics, sinkFlushContext); + } + + @Override + public boolean isMaxEventsLimitReached(final long maxEvents) { + return buffer.size() >= maxEvents; + } + + @Override + public boolean willExceedMaxRequestSizeBytes(final SinkBufferEntry sinkBufferEntry, final long maxRequestSize) { + return currentSize + sinkBufferEntry.getEstimatedSize() > maxRequestSize; + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkFlushContext.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkFlushContext.java new file mode 100644 index 0000000000..53c3ed45be --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkFlushContext.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http.service; + +import org.opensearch.dataprepper.common.sink.SinkFlushContext; +import org.opensearch.dataprepper.model.codec.OutputCodec; +import org.opensearch.dataprepper.model.sink.OutputCodecContext; +import org.opensearch.dataprepper.plugins.sink.http.HttpSinkSender; + +public class HttpSinkFlushContext implements SinkFlushContext { + private final HttpSinkSender httpSender; + private final OutputCodec codec; + private final OutputCodecContext codecContext; + + public HttpSinkFlushContext(final HttpSinkSender httpSender, final OutputCodec codec, final OutputCodecContext codecContext) { + this.httpSender = httpSender; + this.codec = codec; + this.codecContext = codecContext; + } + + public HttpSinkSender getHttpSender() { + return httpSender; + } + + public OutputCodec getCodec() { + return codec; + } + + public OutputCodecContext getCodecContext() { + return codecContext; + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkFlushableBuffer.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkFlushableBuffer.java new file mode 100644 index 0000000000..286cf58e6a --- /dev/null +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkFlushableBuffer.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http.service; + +import org.opensearch.dataprepper.common.sink.DefaultSinkFlushResult; +import org.opensearch.dataprepper.common.sink.SinkBufferEntry; +import org.opensearch.dataprepper.common.sink.SinkFlushContext; +import org.opensearch.dataprepper.common.sink.SinkFlushResult; +import org.opensearch.dataprepper.common.sink.SinkFlushableBuffer; +import org.opensearch.dataprepper.common.sink.SinkMetrics; +import org.opensearch.dataprepper.model.codec.OutputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.sink.OutputCodecContext; +import org.opensearch.dataprepper.plugins.sink.http.HttpEndpointResponse; +import org.opensearch.dataprepper.plugins.sink.http.HttpSinkSender; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +public class HttpSinkFlushableBuffer implements SinkFlushableBuffer { + private final List buffer; + private final HttpSinkFlushContext flushContext; + private final SinkMetrics sinkMetrics; + + public HttpSinkFlushableBuffer(final List buffer, final SinkMetrics sinkMetrics, final SinkFlushContext flushContext) { + this.buffer = buffer; + this.sinkMetrics = sinkMetrics; + this.flushContext = (HttpSinkFlushContext) flushContext; + } + + @Override + public SinkFlushResult flush() { + if (buffer.isEmpty()) { + return null; + } + + final HttpSinkSender httpSender = flushContext.getHttpSender(); + final OutputCodec codec = flushContext.getCodec(); + final OutputCodecContext codecContext = flushContext.getCodecContext(); + final List events = getEvents(); + + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + final OutputCodec.Writer writer = codec.createWriter(outputStream, buffer.get(0).getEvent(), codecContext); + for (final SinkBufferEntry entry : buffer) { + writer.writeEvent(entry.getEvent()); + } + writer.complete(); + + final byte[] data = outputStream.toByteArray(); + final HttpEndpointResponse response = httpSender.send(data); + + if (response.getStatusCode() >= 200 && response.getStatusCode() < 300) { + sinkMetrics.incrementRequestsSuccessCounter(1); + sinkMetrics.incrementEventsSuccessCounter(events.size()); + return null; + } else { + sinkMetrics.incrementRequestsFailedCounter(1); + sinkMetrics.incrementEventsFailedCounter(events.size()); + return new DefaultSinkFlushResult(events, response.getStatusCode(), new RuntimeException(response.getErrMessage())); + } + } catch (IOException e) { + sinkMetrics.incrementRequestsFailedCounter(1); + sinkMetrics.incrementEventsFailedCounter(events.size()); + return new DefaultSinkFlushResult(events, 0, e); + } + } + + @Override + public List getEvents() { + return buffer.stream() + .map(SinkBufferEntry::getEvent) + .collect(Collectors.toList()); + } +} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkService.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkService.java index c392bf4ae0..2c654ad7e1 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkService.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkService.java @@ -1,380 +1,103 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.service; -import io.micrometer.core.instrument.Counter; -import org.apache.hc.client5.http.config.RequestConfig; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.io.HttpClientConnectionManager; -import org.apache.hc.client5.http.protocol.HttpClientContext; -import org.apache.hc.core5.http.ContentType; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; -import org.apache.hc.core5.util.Timeout; -import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.common.sink.DefaultSinkOutputStrategy; +import org.opensearch.dataprepper.common.sink.ReentrantLockStrategy; +import org.opensearch.dataprepper.common.sink.SinkBufferEntry; +import org.opensearch.dataprepper.common.sink.SinkMetrics; import org.opensearch.dataprepper.model.codec.OutputCodec; -import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.EventHandle; +import org.opensearch.dataprepper.model.pipeline.HeadlessPipeline; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.OutputCodecContext; -import org.opensearch.dataprepper.model.types.ByteCount; - -import org.opensearch.dataprepper.plugins.accumulator.Buffer; -import org.opensearch.dataprepper.plugins.accumulator.BufferFactory; -import org.opensearch.dataprepper.plugins.sink.http.HttpEndPointResponse; -import org.opensearch.dataprepper.plugins.sink.http.OAuthAccessTokenManager; -import org.opensearch.dataprepper.plugins.sink.ThresholdValidator; - -import org.opensearch.dataprepper.plugins.sink.http.certificate.CertificateProviderFactory; -import org.opensearch.dataprepper.plugins.sink.http.certificate.HttpClientSSLConnectionManager; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AuthTypeOptions; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HTTPMethodOptions; +import org.opensearch.dataprepper.plugins.sink.http.HttpSinkSender; import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.opensearch.dataprepper.plugins.sink.http.dlq.DlqPushHandler; -import org.opensearch.dataprepper.plugins.sink.http.dlq.FailedDlqData; -import org.opensearch.dataprepper.plugins.sink.http.handler.BasicAuthHttpSinkHandler; -import org.opensearch.dataprepper.plugins.sink.http.handler.BearerTokenAuthHttpSinkHandler; -import org.opensearch.dataprepper.plugins.sink.http.handler.HttpAuthOptions; -import org.opensearch.dataprepper.plugins.sink.http.FailedHttpResponseInterceptor; -import org.opensearch.dataprepper.plugins.sink.http.handler.MultiAuthHttpSinkHandler; -import org.opensearch.dataprepper.plugins.sink.http.util.HttpSinkUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.OutputStream; - -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * This service class contains logic for sending data to Http Endpoints - */ -public class HttpSinkService { - - private static final Logger LOG = LoggerFactory.getLogger(HttpSinkService.class); - - public static final String HTTP_SINK_RECORDS_SUCCESS_COUNTER = "httpSinkRecordsSuccessPushToEndPoint"; - - public static final String HTTP_SINK_RECORDS_FAILED_COUNTER = "httpSinkRecordsFailedToPushEndPoint"; - - private final Collection bufferedEventHandles; - - private final HttpSinkConfiguration httpSinkConfiguration; - - private final BufferFactory bufferFactory; - - private final Map httpAuthOptions; - - private DlqPushHandler dlqPushHandler; - - private final PluginSetting pluginSetting; - - private final Lock reentrantLock; - - private final HttpClientBuilder httpClientBuilder; - - private final int maxEvents; - - private final ByteCount maxBytes; - - private final long maxCollectionDuration; - - private final Counter httpSinkRecordsSuccessCounter; - - private final Counter httpSinkRecordsFailedCounter; - - private final OAuthAccessTokenManager oAuthAccessTokenManager; - - private CertificateProviderFactory certificateProviderFactory; - - private WebhookService webhookService; - - private HttpClientConnectionManager httpClientConnectionManager; - - private Buffer currentBuffer; - - private final PluginSetting httpPluginSetting; - - private MultiAuthHttpSinkHandler multiAuthHttpSinkHandler; +public class HttpSinkService extends DefaultSinkOutputStrategy { + static final String PLUGIN_NAME = "http"; + private final List> dlqRecords; + private final PipelineDescription pipelineDescription; private final OutputCodec codec; - private final OutputCodecContext codecContext; + private HeadlessPipeline dlqPipeline; + private boolean dropIfNoDLQConfigured; public HttpSinkService(final HttpSinkConfiguration httpSinkConfiguration, - final BufferFactory bufferFactory, - final DlqPushHandler dlqPushHandler, - final PluginSetting pluginSetting, - final WebhookService webhookService, - final HttpClientBuilder httpClientBuilder, - final PluginMetrics pluginMetrics, - final PluginSetting httpPluginSetting, + final SinkMetrics sinkMetrics, + final HttpSinkSender httpSender, + final PipelineDescription pipelineDescription, final OutputCodec codec, final OutputCodecContext codecContext) { - - this.httpSinkConfiguration = httpSinkConfiguration; - this.bufferFactory = bufferFactory; - this.dlqPushHandler = dlqPushHandler; - this.pluginSetting = pluginSetting; - this.reentrantLock = new ReentrantLock(); - this.webhookService = webhookService; - this.bufferedEventHandles = new LinkedList<>(); - this.httpClientBuilder = httpClientBuilder; - this.maxEvents = httpSinkConfiguration.getThresholdOptions().getEventCount(); - this.maxBytes = httpSinkConfiguration.getThresholdOptions().getMaximumSize(); - this.maxCollectionDuration = httpSinkConfiguration.getThresholdOptions().getEventCollectTimeOut().getSeconds(); - this.httpPluginSetting = httpPluginSetting; - this.oAuthAccessTokenManager = new OAuthAccessTokenManager(); - - if ((!httpSinkConfiguration.isInsecureSkipVerify()) || (httpSinkConfiguration.useAcmCertForSSL())) { - this.certificateProviderFactory = new CertificateProviderFactory(httpSinkConfiguration); - this.httpClientConnectionManager = new HttpClientSSLConnectionManager() - .createHttpClientConnectionManager(httpSinkConfiguration, certificateProviderFactory); - } - else{ - try { - this.httpClientConnectionManager = new HttpClientSSLConnectionManager().createHttpClientConnectionManagerWithoutValidation(); - }catch(NoSuchAlgorithmException | KeyStoreException | KeyManagementException ex){ - LOG.error("Exception while insecure_skip_verify is true ",ex); - } - } - this.httpAuthOptions = buildAuthHttpSinkObjectsByConfig(httpSinkConfiguration); - this.httpSinkRecordsSuccessCounter = pluginMetrics.counter(HTTP_SINK_RECORDS_SUCCESS_COUNTER); - this.httpSinkRecordsFailedCounter = pluginMetrics.counter(HTTP_SINK_RECORDS_FAILED_COUNTER); - this.codec= codec; + super(new ReentrantLockStrategy(), + new HttpSinkBuffer( + httpSinkConfiguration.getThresholdOptions().getMaxEvents(), + httpSinkConfiguration.getThresholdOptions().getMaxRequestSize().getBytes(), + httpSinkConfiguration.getThresholdOptions().getFlushTimeOut().toMillis(), + new HttpSinkBufferWriter(sinkMetrics)), + new HttpSinkFlushContext(httpSender, codec, codecContext), + sinkMetrics); + this.dlqRecords = new ArrayList<>(); + this.pipelineDescription = pipelineDescription; + this.codec = codec; this.codecContext = codecContext; + this.dropIfNoDLQConfigured = false; } - /** - * This method process buffer records and send to Http End points based on configured codec - * @param records Collection of Event - */ - public void output(Collection> records) { - reentrantLock.lock(); - if (currentBuffer == null) { - this.currentBuffer = bufferFactory.getBuffer(); - } - try { - OutputStream outputStream = currentBuffer.getOutputStream(); - records.forEach(record -> { - try { - final Event event = record.getData(); - if(currentBuffer.getEventCount() == 0) { - codec.start(outputStream,event , codecContext); - } - codec.writeEvent(event, outputStream); - int count = currentBuffer.getEventCount() +1; - currentBuffer.setEventCount(count); - - bufferedEventHandles.add(event.getEventHandle()); - if (ThresholdValidator.checkThresholdExceed(currentBuffer, maxEvents, maxBytes, maxCollectionDuration)) { - codec.complete(outputStream); - final HttpEndPointResponse failedHttpEndPointResponses = pushToEndPoint(getCurrentBufferData(currentBuffer)); - if (failedHttpEndPointResponses != null) { - logFailedData(failedHttpEndPointResponses); - releaseEventHandles(Boolean.FALSE); - } else { - LOG.info("data pushed to the end point successfully"); - releaseEventHandles(Boolean.TRUE); - } - currentBuffer = bufferFactory.getBuffer(); - }} - catch (IOException e) { - throw new RuntimeException(e); - }}); - }finally { - reentrantLock.unlock(); - } - } - - private byte[] getCurrentBufferData(final Buffer currentBuffer) { - try { - return currentBuffer.getSinkBufferData(); - } catch (IOException e) { - throw new RuntimeException(e); - } + @Override + public SinkBufferEntry getSinkBufferEntry(final Event event) throws Exception { + return new HttpSinkBufferEntry(event, codec, codecContext); } - /** - * * This method logs Failed Data to DLQ and Webhook - * @param endPointResponses HttpEndPointResponses. - */ - private void logFailedData(final HttpEndPointResponse endPointResponses) { - FailedDlqData failedDlqData = - FailedDlqData.builder() - .withUrl(endPointResponses.getUrl()) - .withMessage(endPointResponses.getErrMessage()) - .withStatus(endPointResponses.getStatusCode()).build(); - - LOG.info("Failed to push the data. Failed DLQ Data: {}",failedDlqData); - - logFailureForDlqObjects(failedDlqData); - if(Objects.nonNull(webhookService)){ - logFailureForWebHook(failedDlqData); - } + public void setDlqPipeline(final HeadlessPipeline pipeline) { + this.dlqPipeline = pipeline; } - private void releaseEventHandles(final boolean result) { - for (EventHandle eventHandle : bufferedEventHandles) { - eventHandle.release(result); + @Override + public void flushDlqList() { + if (dlqRecords.isEmpty()) { + return; } - bufferedEventHandles.clear(); - } - - /** - * * This method pushes bufferData to configured HttpEndPoints - * @param currentBufferData bufferData. - */ - private HttpEndPointResponse pushToEndPoint(final byte[] currentBufferData) { - HttpEndPointResponse httpEndPointResponses = null; - final ClassicRequestBuilder classicHttpRequestBuilder = - httpAuthOptions.get(httpSinkConfiguration.getUrl()).getClassicHttpRequestBuilder(); - classicHttpRequestBuilder.setEntity(currentBufferData, ContentType.APPLICATION_JSON); - try { - if(AuthTypeOptions.BEARER_TOKEN.equals(httpSinkConfiguration.getAuthType())) - accessTokenIfExpired(httpSinkConfiguration.getAuthentication().getBearerTokenOptions().getTokenExpired(),httpSinkConfiguration.getUrl()); - httpAuthOptions.get(httpSinkConfiguration.getUrl()).getHttpClientBuilder().build() - .execute(classicHttpRequestBuilder.build(), HttpClientContext.create()); - LOG.info("No of Records successfully pushed to endpoint {}", httpSinkConfiguration.getUrl() +" " + currentBuffer.getEventCount()); - httpSinkRecordsSuccessCounter.increment(currentBuffer.getEventCount()); - } catch (IOException e) { - httpSinkRecordsFailedCounter.increment(currentBuffer.getEventCount()); - LOG.info("No of Records failed to push endpoint {}",currentBuffer.getEventCount()); - LOG.error("Exception while pushing buffer data to end point. URL : {}, Exception : ", httpSinkConfiguration.getUrl(), e); - httpEndPointResponses = new HttpEndPointResponse(httpSinkConfiguration.getUrl(), HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + if (dlqPipeline != null) { + dlqPipeline.sendEvents(dlqRecords); } - return httpEndPointResponses; + dlqRecords.clear(); } - /** - * * This method sends Failed objects to DLQ - * @param failedDlqData FailedDlqData. - */ - private void logFailureForDlqObjects(final FailedDlqData failedDlqData){ - dlqPushHandler.perform(httpPluginSetting, failedDlqData); - } - - /** - * * This method push Failed objects to Webhook - * @param failedDlqData FailedDlqData. - */ - private void logFailureForWebHook(final FailedDlqData failedDlqData){ - webhookService.pushWebhook(failedDlqData); - } - - /** - * * This method gets Auth Handler classes based on configuration - * @param authType AuthTypeOptions. - * @param authOptions HttpAuthOptions.Builder. - */ - private HttpAuthOptions getAuthHandlerByConfig(final AuthTypeOptions authType, - final HttpAuthOptions.Builder authOptions){ - switch(authType) { - case HTTP_BASIC: - multiAuthHttpSinkHandler = new BasicAuthHttpSinkHandler( - httpSinkConfiguration.getAuthentication().getHttpBasic().getUsername(), - httpSinkConfiguration.getAuthentication().getHttpBasic().getPassword(), - httpClientConnectionManager); - break; - case BEARER_TOKEN: - multiAuthHttpSinkHandler = new BearerTokenAuthHttpSinkHandler( - httpSinkConfiguration.getAuthentication().getBearerTokenOptions(), - httpClientConnectionManager, oAuthAccessTokenManager); - break; - case UNAUTHENTICATED: - default: - return authOptions.setHttpClientBuilder(httpClientBuilder - .setConnectionManager(httpClientConnectionManager) - .addResponseInterceptorLast(new FailedHttpResponseInterceptor(authOptions.getUrl()))).build(); - } - return multiAuthHttpSinkHandler.authenticate(authOptions); - } - - /** - * * This method build HttpAuthOptions class based on configurations - * @param httpSinkConfiguration HttpSinkConfiguration. - */ - private Map buildAuthHttpSinkObjectsByConfig(final HttpSinkConfiguration httpSinkConfiguration){ - final Map authMap = new HashMap<>(); - - final HTTPMethodOptions httpMethod = httpSinkConfiguration.getHttpMethod(); - final AuthTypeOptions authType = httpSinkConfiguration.getAuthType(); - final String proxyUrlString = httpSinkConfiguration.getProxy(); - final ClassicRequestBuilder classicRequestBuilder = buildRequestByHTTPMethodType(httpMethod).setUri(httpSinkConfiguration.getUrl()); - - - - if(Objects.nonNull(httpSinkConfiguration.getCustomHeaderOptions())) - addCustomHeaders(classicRequestBuilder,httpSinkConfiguration.getCustomHeaderOptions()); - - if(httpSinkConfiguration.isAwsSigv4() && httpSinkConfiguration.isValidAWSUrl()){ - classicRequestBuilder.addHeader("x-amz-content-sha256","required"); - } - - if(Objects.nonNull(proxyUrlString)) { - httpClientBuilder.setProxy(HttpSinkUtil.getHttpHostByURL(HttpSinkUtil.getURLByUrlString(proxyUrlString))); - LOG.info("sending data via proxy {}",proxyUrlString); - } - if(httpSinkConfiguration.getRequestTimout() != null) { - httpClientBuilder.setDefaultRequestConfig(RequestConfig.custom().setConnectionRequestTimeout(Timeout.ofMilliseconds(httpSinkConfiguration.getRequestTimout().toMillis())).build()); - } - - final HttpAuthOptions.Builder authOptions = new HttpAuthOptions.Builder() - .setUrl(httpSinkConfiguration.getUrl()) - .setClassicHttpRequestBuilder(classicRequestBuilder) - .setHttpClientBuilder(httpClientBuilder); - - authMap.put(httpSinkConfiguration.getUrl(),getAuthHandlerByConfig(authType,authOptions)); - return authMap; - } - - /** - * * This method adds SageMakerHeaders as custom Header in the request - * @param classicRequestBuilder ClassicRequestBuilder. - * @param customHeaderOptions CustomHeaderOptions . - */ - private void addCustomHeaders(final ClassicRequestBuilder classicRequestBuilder, - final Map> customHeaderOptions) { - - customHeaderOptions.forEach((k, v) -> classicRequestBuilder.addHeader(k,v.toString())); - } - - /** - * * builds ClassicRequestBuilder based on configured HttpMethod - * @param httpMethodOptions Http Method. - */ - private ClassicRequestBuilder buildRequestByHTTPMethodType(final HTTPMethodOptions httpMethodOptions) { - final ClassicRequestBuilder classicRequestBuilder; - switch (httpMethodOptions) { - case PUT: - classicRequestBuilder = ClassicRequestBuilder.put(); - break; - case POST: - default: - classicRequestBuilder = ClassicRequestBuilder.post(); - break; + @Override + public void addFailedEventsToDlq(final List failedEvents, final Throwable ex, final int statusCode) { + for (final Event event : failedEvents) { + if (dlqPipeline == null) { + event.getEventHandle().release(dropIfNoDLQConfigured); + continue; + } + event.updateFailureMetadata() + .with("statusCode", statusCode) + .withPluginName(PLUGIN_NAME) + .withPipelineName(pipelineDescription.getPipelineName()); + if (ex != null) { + event.updateFailureMetadata() + .withErrorMessage(ex.getMessage()); + } + dlqRecords.add(new Record<>(event)); } - return classicRequestBuilder; } - private void accessTokenIfExpired(final Integer tokenExpired,final String url){ - if(oAuthAccessTokenManager.isTokenExpired(tokenExpired)) { - httpAuthOptions.get(url).getClassicHttpRequestBuilder() - .setHeader(BearerTokenAuthHttpSinkHandler.AUTHORIZATION, oAuthAccessTokenManager.getAccessToken(httpSinkConfiguration.getAuthentication().getBearerTokenOptions())); - } + public void output(final Collection> records) { + execute(records); } - } + diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/WebhookService.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/WebhookService.java deleted file mode 100644 index 40143718d2..0000000000 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/service/WebhookService.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.service; - -import io.micrometer.core.instrument.Counter; -import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.protocol.HttpClientContext; -import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; -import org.apache.hc.core5.util.TimeValue; -import org.opensearch.dataprepper.metrics.PluginMetrics; - -import org.opensearch.dataprepper.plugins.sink.http.FailedHttpResponseInterceptor; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.opensearch.dataprepper.plugins.sink.http.dlq.FailedDlqData; -import org.opensearch.dataprepper.plugins.sink.http.util.HttpSinkUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URL; - -public class WebhookService { - - private static final Logger LOG = LoggerFactory.getLogger(WebhookService.class); - - public static final String HTTP_SINK_SUCCESS_WEBHOOKS = "httpSinkSuccessWebhooks"; - - public static final String HTTP_SINK_FAILED_WEBHOOKS = "httpSinkFailedWebhooks"; - - private final HttpClientBuilder httpClientBuilder; - - private final Counter httpSinkWebhookSuccessCounter; - - private final Counter httpSinkWebhookFailedCounter; - - private final DefaultHttpRequestRetryStrategy httpRequestRetryStrategy; - - private URL url; - - public WebhookService(final String url, - final HttpClientBuilder httpClientBuilder, - final PluginMetrics pluginMetrics, - final HttpSinkConfiguration httpSinkConfiguration){ - this.httpClientBuilder = httpClientBuilder; - this.url = HttpSinkUtil.getURLByUrlString(url); - this.httpSinkWebhookSuccessCounter = pluginMetrics.counter(HTTP_SINK_SUCCESS_WEBHOOKS); - this.httpSinkWebhookFailedCounter = pluginMetrics.counter(HTTP_SINK_FAILED_WEBHOOKS); - this.httpRequestRetryStrategy = new DefaultHttpRequestRetryStrategy(httpSinkConfiguration.getMaxUploadRetries(), - TimeValue.of(httpSinkConfiguration.getHttpRetryInterval())); - } - - /** - * * It sends failed dlq data to configured webhook url - * @param failedDlqData Failed Dlq data. - */ - public void pushWebhook(final FailedDlqData failedDlqData) { - final HttpHost targetHost; - final CloseableHttpResponse webhookResp; - targetHost = HttpSinkUtil.getHttpHostByURL(url); - final ClassicRequestBuilder classicHttpRequestBuilder = - ClassicRequestBuilder.post().setEntity(failedDlqData.toString()).setUri(url.toString()); - try { - webhookResp = httpClientBuilder - .setRetryStrategy(new DefaultHttpRequestRetryStrategy()) - .addResponseInterceptorLast(new FailedHttpResponseInterceptor(url.toString())) - .build() - .execute(targetHost, classicHttpRequestBuilder.build(), HttpClientContext.create()); - httpSinkWebhookSuccessCounter.increment(); - } catch (IOException e) { - httpSinkWebhookFailedCounter.increment(); - LOG.error("Exception while pushing webhook: ",e); - } - } -} diff --git a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtil.java b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtil.java index 13fbaff663..b542da845e 100644 --- a/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtil.java +++ b/data-prepper-plugins/http-sink/src/main/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtil.java @@ -1,7 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.util; import org.apache.hc.core5.http.HttpHost; diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/AwsAuthenticationDecoratorTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/AwsAuthenticationDecoratorTest.java new file mode 100644 index 0000000000..30ec51d3de --- /dev/null +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/AwsAuthenticationDecoratorTest.java @@ -0,0 +1,212 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.sink.http; + +import com.linecorp.armeria.common.HttpRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.aws.api.AwsConfig; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class AwsAuthenticationDecoratorTest { + + private AwsCredentialsSupplier awsCredentialsSupplier; + private AwsCredentialsProvider awsCredentialsProvider; + private AwsConfig awsConfig; + + private static final String TEST_URL = "https://example.com/test"; + private static final byte[] TEST_PAYLOAD = "{\"key\":\"value\"}".getBytes(StandardCharsets.UTF_8); + private static final String TEST_ROLE_ARN = "arn:aws:iam::123456789012:role/TestRole"; + private static final String TEST_EXTERNAL_ID = "test-external-id"; + private static final Region TEST_REGION = Region.US_EAST_1; + private static final Map TEST_HEADER_OVERRIDES = Map.of("headerKey", "headerValue"); + private static final String TEST_SERVICE_NAME = "execute-api"; + + @BeforeEach + void setUp() { + awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); + awsCredentialsProvider = mock(AwsCredentialsProvider.class); + awsConfig = mock(AwsConfig.class); + + when(awsConfig.getAwsRegion()).thenReturn(TEST_REGION); + when(awsConfig.getAwsStsRoleArn()).thenReturn(TEST_ROLE_ARN); + when(awsConfig.getAwsStsExternalId()).thenReturn(TEST_EXTERNAL_ID); + when(awsConfig.getAwsStsHeaderOverrides()).thenReturn(TEST_HEADER_OVERRIDES); + when(awsCredentialsSupplier.getProvider(any(AwsCredentialsOptions.class))).thenReturn(awsCredentialsProvider); + when(awsCredentialsProvider.resolveCredentials()) + .thenReturn(AwsBasicCredentials.create("testAccessKey", "testSecretKey")); + } + + private AwsAuthenticationDecorator createObjectUnderTest() { + return new AwsAuthenticationDecorator(awsCredentialsSupplier, awsConfig, TEST_SERVICE_NAME); + } + + private AwsAuthenticationDecorator createObjectUnderTest(final String serviceName) { + return new AwsAuthenticationDecorator(awsCredentialsSupplier, awsConfig, serviceName); + } + + @Test + void test_constructor_passes_correct_credentials_options() { + final ArgumentCaptor optionsCaptor = + ArgumentCaptor.forClass(AwsCredentialsOptions.class); + + createObjectUnderTest(); + + verify(awsCredentialsSupplier).getProvider(optionsCaptor.capture()); + final AwsCredentialsOptions capturedOptions = optionsCaptor.getValue(); + + assertThat(capturedOptions.getRegion(), equalTo(TEST_REGION)); + assertThat(capturedOptions.getStsRoleArn(), equalTo(TEST_ROLE_ARN)); + assertThat(capturedOptions.getStsExternalId(), equalTo(TEST_EXTERNAL_ID)); + assertThat(capturedOptions.getStsHeaderOverrides(), equalTo(TEST_HEADER_OVERRIDES)); + } + + @Test + void test_buildRequest_returns_non_null_http_request() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + assertThat(request, notNullValue()); + } + + @Test + void test_buildRequest_contains_authorization_header() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + assertTrue(request.headers().contains("Authorization"), "Request should contain Authorization header"); + final String authHeader = request.headers().get("Authorization"); + assertTrue(authHeader.startsWith("AWS4-HMAC-SHA256"), "Authorization header should use AWS4-HMAC-SHA256 signing"); + } + + @Test + void test_buildRequest_signs_with_correct_region() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + final String authHeader = request.headers().get("Authorization"); + assertTrue(authHeader.contains(TEST_REGION.id()), "Authorization header should contain the configured region"); + } + + @Test + void test_buildRequest_signs_with_configured_service_name() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + final String authHeader = request.headers().get("Authorization"); + assertTrue(authHeader.contains(TEST_SERVICE_NAME), "Authorization header should contain the service signing name"); + } + + @Test + void test_buildRequest_signs_with_custom_service_name() { + final String customServiceName = "osis"; + final AwsAuthenticationDecorator decorator = createObjectUnderTest(customServiceName); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + final String authHeader = request.headers().get("Authorization"); + assertTrue(authHeader.contains(customServiceName), "Authorization header should contain the custom service signing name"); + } + + @Test + void test_buildRequest_contains_content_sha256_header() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + assertTrue(request.headers().contains("x-amz-content-sha256"), "Request should contain x-amz-content-sha256 header"); + assertThat(request.headers().get("x-amz-content-sha256").length(), equalTo(64)); + } + + @Test + void test_buildRequest_includes_custom_headers() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + final Map> customHeaders = Map.of( + "X-Custom-Header", List.of("custom-value"), + "X-Another", List.of("val1", "val2") + ); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, customHeaders); + + assertTrue(request.headers().contains("X-Custom-Header"), "Request should contain custom header"); + assertThat(request.headers().get("X-Custom-Header"), equalTo("custom-value")); + } + + @Test + void test_buildRequest_preserves_url_path() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + assertThat(request.path(), equalTo("/test")); + } + + @Test + void test_buildRequest_preserves_url_authority() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + assertThat(request.authority(), equalTo("example.com")); + } + + @Test + void test_buildRequest_resolves_credentials_on_each_call() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + verify(awsCredentialsProvider, org.mockito.Mockito.times(2)).resolveCredentials(); + } + + @Test + void test_buildRequest_with_null_custom_headers() { + final AwsAuthenticationDecorator decorator = createObjectUnderTest(); + + final HttpRequest request = decorator.buildRequest(TEST_URL, TEST_PAYLOAD, null); + + assertThat(request, notNullValue()); + } + + @Test + void test_constructor_with_null_supplier_throws_exception() { + assertThrows(NullPointerException.class, () -> new AwsAuthenticationDecorator(null, awsConfig, TEST_SERVICE_NAME)); + } + + @Test + void test_constructor_with_null_config_throws_exception() { + assertThrows(NullPointerException.class, () -> new AwsAuthenticationDecorator(awsCredentialsSupplier, null, TEST_SERVICE_NAME)); + } +} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptorTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptorTest.java index a02e3685a2..b8aa6c0b4f 100644 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptorTest.java +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/FailedHttpResponseInterceptorTest.java @@ -1,7 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http; import org.apache.hc.core5.http.EntityDetails; diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkTest.java index 430ae9faf8..896b7a9239 100644 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkTest.java +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/HttpSinkTest.java @@ -1,13 +1,20 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.codec.OutputCodec; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginModel; @@ -16,14 +23,12 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.sink.SinkContext; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AuthTypeOptions; -import org.opensearch.dataprepper.plugins.accumulator.BufferTypeOptions; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AwsAuthenticationOptions; +import org.opensearch.dataprepper.model.types.ByteCount; +import org.opensearch.dataprepper.aws.api.AwsConfig; import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; import org.opensearch.dataprepper.plugins.sink.http.configuration.ThresholdOptions; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HTTPMethodOptions; -import org.opensearch.dataprepper.plugins.sink.http.handler.HttpAuthOptions; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -31,12 +36,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class HttpSinkTest { - HTTPSink httpSink; + HttpSink httpSink; private PluginSetting pluginSetting; private PluginFactory pluginFactory; @@ -51,11 +57,11 @@ public class HttpSinkTest { private ThresholdOptions thresholdOptions; - private AwsAuthenticationOptions awsAuthenticationOptions; + private AwsConfig awsConfig; private OutputCodec codec; - private HttpAuthOptions httpAuthOptions; + private PluginMetrics pluginMetrics; @BeforeEach @@ -68,33 +74,31 @@ void setUp() { thresholdOptions = mock(ThresholdOptions.class); sinkContext = mock(SinkContext.class); codec = mock(OutputCodec.class); - httpAuthOptions = mock(HttpAuthOptions.class); - awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); + awsConfig = mock(AwsConfig.class); + pluginMetrics = mock(PluginMetrics.class); when(pluginSetting.getPipelineName()).thenReturn("log-pipeline"); - PluginModel codecConfiguration = new PluginModel("http", new HashMap<>()); + when(pipelineDescription.getPipelineName()).thenReturn("log-pipeline"); + PluginModel codecConfiguration = new PluginModel("ndjson", new HashMap<>()); when(httpSinkConfiguration.getCodec()).thenReturn(codecConfiguration); - when(httpSinkConfiguration.getBufferType()).thenReturn(BufferTypeOptions.LOCALFILE); - when(httpAuthOptions.getUrl()).thenReturn("http://localhost:8080"); - when(httpSinkConfiguration.getHttpMethod()).thenReturn(HTTPMethodOptions.POST); - when(httpSinkConfiguration.getAuthType()).thenReturn(AuthTypeOptions.UNAUTHENTICATED); + when(pluginFactory.loadPlugin(any(), any())).thenReturn(codec); + when(httpSinkConfiguration.getUrl()).thenReturn("http://localhost:8080"); + when(httpSinkConfiguration.getConnectionTimeout()).thenReturn(Duration.ofSeconds(10)); Map dlqSetting = new HashMap<>(); dlqSetting.put("bucket", "dlq.test"); dlqSetting.put("key_path_prefix", "\\dlq"); PluginModel dlq = new PluginModel("s3",dlqSetting); - when(httpSinkConfiguration.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); - when(httpSinkConfiguration.getDlqStsRoleARN()).thenReturn("arn:aws:iam::1234567890:role/app-test"); - when(httpSinkConfiguration.getDlqStsRegion()).thenReturn("ap-south-1"); - when(httpSinkConfiguration.getDlq()).thenReturn(dlq); + when(httpSinkConfiguration.getAwsConfig()).thenReturn(awsConfig); when(httpSinkConfiguration.getThresholdOptions()).thenReturn(thresholdOptions); - when(thresholdOptions.getEventCount()).thenReturn(10); - when(httpSinkConfiguration.getDlqFile()).thenReturn("\\dlq"); + when(thresholdOptions.getMaxEvents()).thenReturn(10); + when(thresholdOptions.getMaxRequestSize()).thenReturn(ByteCount.parse("50mb")); + when(thresholdOptions.getFlushTimeOut()).thenReturn(Duration.ofSeconds(10)); when(sinkContext.getIncludeKeys()).thenReturn(new ArrayList<>()); when(sinkContext.getExcludeKeys()).thenReturn(new ArrayList<>()); } - private HTTPSink createObjectUnderTest() { - return new HTTPSink(pluginSetting, httpSinkConfiguration, pluginFactory, pipelineDescription, sinkContext, - awsCredentialsSupplier); + private HttpSink createObjectUnderTest() { + return new HttpSink(pluginSetting, httpSinkConfiguration, pluginFactory, pipelineDescription, sinkContext, + awsCredentialsSupplier, pluginMetrics); } @Test void test_http_sink_plugin_isReady_positive() { diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/certificate/CertificateProviderFactoryTest.java deleted file mode 100644 index d70e5fe120..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/certificate/CertificateProviderFactoryTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.dataprepper.plugins.sink.http.certificate; - -import org.hamcrest.core.IsInstanceOf; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.acm.ACMCertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.file.FileCertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.s3.S3CertificateProvider; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AwsAuthenticationOptions; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import software.amazon.awssdk.regions.Region; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class CertificateProviderFactoryTest { - private final String TEST_SSL_CERTIFICATE_FILE = getClass().getClassLoader().getResource("test_cert.crt").getFile(); - private final String TEST_SSL_KEY_FILE = getClass().getClassLoader().getResource("test_decrypted_key.key").getFile(); - - private HttpSinkConfiguration httpSinkConfiguration; - - private AwsAuthenticationOptions awsAuthenticationOptions; - private CertificateProviderFactory certificateProviderFactory; - - @BeforeEach - void setUp() { - httpSinkConfiguration = mock(HttpSinkConfiguration.class); - awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); - } - - @Test - void getCertificateProviderFileCertificateProviderSuccess() { - when(httpSinkConfiguration.isInsecureSkipVerify()).thenReturn(true); - when(httpSinkConfiguration.getSslCertificateFile()).thenReturn(TEST_SSL_CERTIFICATE_FILE); - when(httpSinkConfiguration.getSslKeyFile()).thenReturn(TEST_SSL_KEY_FILE); - - certificateProviderFactory = new CertificateProviderFactory(httpSinkConfiguration); - final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); - - assertThat(certificateProvider, IsInstanceOf.instanceOf(FileCertificateProvider.class)); - } - - @Test - void getCertificateProviderS3ProviderSuccess() { - when(httpSinkConfiguration.isSslCertAndKeyFileInS3()).thenReturn(true); - when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.of("us-east-1")); - when(httpSinkConfiguration.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); - when(httpSinkConfiguration.getSslCertificateFile()).thenReturn("s3://data/certificate/test_cert.crt"); - when(httpSinkConfiguration.getSslKeyFile()).thenReturn("s3://data/certificate/test_decrypted_key.key"); - - certificateProviderFactory = new CertificateProviderFactory(httpSinkConfiguration); - final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); - - assertThat(certificateProvider, IsInstanceOf.instanceOf(S3CertificateProvider.class)); - } - - @Test - void getCertificateProviderAcmProviderSuccess() { - when(httpSinkConfiguration.useAcmCertForSSL()).thenReturn(true); - when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.of("us-east-1")); - when(httpSinkConfiguration.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); - when(httpSinkConfiguration.getAcmCertificateArn()).thenReturn("arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - - certificateProviderFactory = new CertificateProviderFactory(httpSinkConfiguration); - final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); - - assertThat(certificateProvider, IsInstanceOf.instanceOf(ACMCertificateProvider.class)); - } -} \ No newline at end of file diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/certificate/HttpClientSSLConnectionManagerTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/certificate/HttpClientSSLConnectionManagerTest.java deleted file mode 100644 index 96bc679d8a..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/certificate/HttpClientSSLConnectionManagerTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.certificate; - -import org.apache.hc.client5.http.io.HttpClientConnectionManager; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.file.FileCertificateProvider; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class HttpClientSSLConnectionManagerTest { - - private final String TEST_SSL_CERTIFICATE_FILE = getClass().getClassLoader().getResource("test_cert.crt").getFile(); - private final String TEST_SSL_KEY_FILE = getClass().getClassLoader().getResource("test_decrypted_key.key").getFile(); - - HttpClientSSLConnectionManager httpClientSSLConnectionManager; - - private CertificateProviderFactory certificateProviderFactory; - - private HttpSinkConfiguration httpSinkConfiguration; - - @BeforeEach - void setup() throws IOException { - this.httpSinkConfiguration = mock(HttpSinkConfiguration.class); - this.certificateProviderFactory = mock(CertificateProviderFactory.class); - } - - @Test - public void create_httpClientConnectionManager_with_ssl_file_test() { - when(httpSinkConfiguration.getSslCertificateFile()).thenReturn(TEST_SSL_CERTIFICATE_FILE); - when(httpSinkConfiguration.getSslKeyFile()).thenReturn(TEST_SSL_KEY_FILE); - CertificateProvider provider = new FileCertificateProvider(httpSinkConfiguration.getSslCertificateFile(), httpSinkConfiguration.getSslKeyFile()); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(provider); - - CertificateProviderFactory providerFactory = new CertificateProviderFactory(httpSinkConfiguration); - httpClientSSLConnectionManager = new HttpClientSSLConnectionManager(); - HttpClientConnectionManager httpClientConnectionManager = httpClientSSLConnectionManager - .createHttpClientConnectionManager(httpSinkConfiguration, providerFactory); - assertNotNull(httpClientConnectionManager); - } -} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AwsAuthenticationOptionsTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AwsAuthenticationOptionsTest.java deleted file mode 100644 index f65b42ad9c..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/AwsAuthenticationOptionsTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.configuration; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import software.amazon.awssdk.regions.Region; - -import java.util.Collections; -import java.util.Map; -import java.util.UUID; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.MatcherAssert.assertThat; - -class AwsAuthenticationOptionsTest { - private ObjectMapper objectMapper = new ObjectMapper(); - @ParameterizedTest - @ValueSource(strings = {"us-east-1", "us-west-2", "eu-central-1"}) - void getAwsRegion_returns_Region_of(final String regionString) { - final Region expectedRegionObject = Region.of(regionString); - final Map jsonMap = Map.of("region", regionString); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsRegion(), equalTo(expectedRegionObject)); - } - - @Test - void getAwsRegion_returns_null_when_region_is_null() { - final Map jsonMap = Collections.emptyMap(); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsRegion(), nullValue()); - } - - @Test - void getAwsStsRoleArn_returns_value_from_deserialized_JSON() { - final String stsRoleArn = UUID.randomUUID().toString(); - final Map jsonMap = Map.of("sts_role_arn", stsRoleArn); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsStsRoleArn(), equalTo(stsRoleArn)); - } - - @Test - void getAwsStsRoleArn_returns_null_if_not_in_JSON() { - final Map jsonMap = Collections.emptyMap(); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsStsRoleArn(), nullValue()); - } - - @Test - void getAwsStsExternalId_returns_value_from_deserialized_JSON() { - final String stsExternalId = UUID.randomUUID().toString(); - final Map jsonMap = Map.of("sts_external_id", stsExternalId); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsStsExternalId(), equalTo(stsExternalId)); - } - - @Test - void getAwsStsExternalId_returns_null_if_not_in_JSON() { - final Map jsonMap = Collections.emptyMap(); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsStsExternalId(), nullValue()); - } - - @Test - void getAwsStsHeaderOverrides_returns_value_from_deserialized_JSON() { - final Map stsHeaderOverrides = Map.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - final Map jsonMap = Map.of("sts_header_overrides", stsHeaderOverrides); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsStsHeaderOverrides(), equalTo(stsHeaderOverrides)); - } - - @Test - void getAwsStsHeaderOverrides_returns_null_if_not_in_JSON() { - final Map jsonMap = Collections.emptyMap(); - final AwsAuthenticationOptions objectUnderTest = objectMapper.convertValue(jsonMap, AwsAuthenticationOptions.class); - assertThat(objectUnderTest.getAwsStsHeaderOverrides(), nullValue()); - } -} \ No newline at end of file diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfigurationTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfigurationTest.java index 625a3b3e04..3e8fb9fdd9 100644 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfigurationTest.java +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/HttpSinkConfigurationTest.java @@ -1,262 +1,108 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.configuration; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; -import org.hamcrest.CoreMatchers; -import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.types.ByteCount; -import org.opensearch.dataprepper.plugins.accumulator.BufferTypeOptions; import software.amazon.awssdk.regions.Region; +import org.opensearch.dataprepper.aws.api.AwsConfig; import java.util.HashMap; -import java.util.List; import java.util.Map; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.opensearch.dataprepper.plugins.sink.http.configuration.AuthTypeOptions.HTTP_BASIC; -import static org.opensearch.dataprepper.plugins.sink.http.configuration.AuthTypeOptions.UNAUTHENTICATED; public class HttpSinkConfigurationTest { private static final String SINK_YAML = " url: \"http://localhost:8080/test\"\n" + - " proxy: test-proxy\n" + " codec:\n" + " ndjson:\n" + - " http_method: \"POST\"\n" + - " auth_type: \"http_basic\"\n" + - " authentication:\n" + - " http_basic:\n" + - " username: \"username\"\n" + - " password: \"vip\"\n" + - " bearer_token:\n" + - " client_id: 0oaafr4j79segrYGC5d7\n" + - " client_secret: fFel-3FutCXAOndezEsOVlght6D6DR4OIt7G5D1_oJ6w0wNoaYtgU17JdyXmGf0M\n" + - " token_url: https://localhost/oauth2/default/v1/token\n" + - " grant_type: client_credentials\n" + - " scope: httpSink\n"+ - " insecure_skip_verify: true\n" + - " dlq_file: \"/your/local/dlq-file\"\n" + - " dlq:\n" + - " s3:\n" + - " bucket: dlq.test\n" + - " key_path_prefix: \\dlq\"\n" + - " ssl_certificate_file: \"/full/path/to/certfile.crt\"\n" + - " ssl_key_file: \"/full/path/to/keyfile.key\"\n" + - " buffer_type: \"in_memory\"\n" + " aws:\n" + " region: \"us-east-2\"\n" + " sts_role_arn: \"arn:aws:iam::895099425785:role/data-prepper-s3source-execution-role\"\n" + " sts_external_id: \"test-external-id\"\n" + " sts_header_overrides: {\"test\": test }\n" + " threshold:\n" + - " event_count: 2000\n" + - " maximum_size: 2mb\n" + - " max_retries: 5\n" + - " aws_sigv4: true\n" + - " webhook_url: \"http://localhost:8080/webhook\"\n" + - " custom_header:\n" + - " X-Amzn-SageMaker-Custom-Attributes: [\"test-attribute\"]\n" + - " X-Amzn-SageMaker-Target-Model: [\"test-target-model\"]\n" + - " X-Amzn-SageMaker-Target-Variant: [\"test-target-variant\"]\n" + - " X-Amzn-SageMaker-Target-Container-Hostname: [\"test-container-host\"]\n" + - " X-Amzn-SageMaker-Inference-Id: [\"test-interface-id\"]\n" + - " X-Amzn-SageMaker-Enable-Explanations: [\"test-explanation\"]"; + " max_events: 2000\n" + + " max_request_size: 2mb\n" + + " max_retries: 5\n"; private ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS)); - @Test - void default_worker_test() { - MatcherAssert.assertThat(new HttpSinkConfiguration().getWorkers(), CoreMatchers.equalTo(1)); - } + @Test void default_codec_test() { assertNull(new HttpSinkConfiguration().getCodec()); } - @Test - void default_proxy_test() { - assertNull(new HttpSinkConfiguration().getProxy()); - } - - @Test - void default_http_method_test() { - assertThat(new HttpSinkConfiguration().getHttpMethod(), CoreMatchers.equalTo(HTTPMethodOptions.POST)); - } - - @Test - void default_auth_type_test() { - assertThat(new HttpSinkConfiguration().getAuthType(), equalTo(UNAUTHENTICATED)); - } - @Test void get_url_test() { assertThat(new HttpSinkConfiguration().getUrl(), equalTo(null)); } - @Test - void get_authentication_test() { - assertNull(new HttpSinkConfiguration().getAuthentication()); - } - - @Test - void default_ssl_test() { - assertThat(new HttpSinkConfiguration().isInsecureSkipVerify(), equalTo(false)); - } - - @Test - void default_awsSigv4_test() { - assertThat(new HttpSinkConfiguration().isAwsSigv4(), equalTo(false)); - } - - @Test - void get_ssl_certificate_file_test() { - assertNull(new HttpSinkConfiguration().getSslCertificateFile()); - } - - @Test - void get_ssl_key_file_test() { - assertNull(new HttpSinkConfiguration().getSslKeyFile()); - } - - @Test - void default_buffer_type_test() { - assertThat(new HttpSinkConfiguration().getBufferType(), equalTo(BufferTypeOptions.INMEMORY)); - } - @Test void get_threshold_options_test() { - assertNull(new HttpSinkConfiguration().getThresholdOptions()); + assertThat(new HttpSinkConfiguration().getThresholdOptions(), instanceOf(ThresholdOptions.class)); } @Test void default_max_upload_retries_test() { - assertThat(new HttpSinkConfiguration().getMaxUploadRetries(), equalTo(5)); + assertThat(new HttpSinkConfiguration().getMaxUploadRetries(), equalTo(HttpSinkConfiguration.DEFAULT_UPLOAD_RETRIES)); } @Test void get_aws_authentication_options_test() { - assertNull(new HttpSinkConfiguration().getAwsAuthenticationOptions()); - } - - @Test - void get_custom_header_options_test() { - assertNull(new HttpSinkConfiguration().getCustomHeaderOptions()); + assertNull(new HttpSinkConfiguration().getAwsConfig()); } @Test void get_http_retry_interval_test() { assertThat(new HttpSinkConfiguration().getHttpRetryInterval(),equalTo(HttpSinkConfiguration.DEFAULT_HTTP_RETRY_INTERVAL)); } - @Test - void get_acm_private_key_password_test() {assertNull(new HttpSinkConfiguration().getAcmPrivateKeyPassword());} - - @Test - void get_is_ssl_cert_and_key_file_in_s3_test() {assertThat(new HttpSinkConfiguration().isSslCertAndKeyFileInS3(), equalTo(false));} - - @Test - void get_acm_cert_issue_time_out_millis_test() {assertThat(new HttpSinkConfiguration().getAcmCertIssueTimeOutMillis(), equalTo(new Long(HttpSinkConfiguration.DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS)));} @Test void http_sink_pipeline_test_with_provided_config_options() throws JsonProcessingException { final HttpSinkConfiguration httpSinkConfiguration = objectMapper.readValue(SINK_YAML, HttpSinkConfiguration.class); assertThat(httpSinkConfiguration.getUrl(),equalTo("http://localhost:8080/test")); - assertThat(httpSinkConfiguration.getHttpMethod(),equalTo(HTTPMethodOptions.POST)); - assertThat(httpSinkConfiguration.getAuthType(),equalTo(HTTP_BASIC)); - assertThat(httpSinkConfiguration.getBufferType(),equalTo(BufferTypeOptions.INMEMORY)); assertThat(httpSinkConfiguration.getMaxUploadRetries(),equalTo(5)); - assertThat(httpSinkConfiguration.getProxy(),equalTo("test-proxy")); - assertThat(httpSinkConfiguration.getSslCertificateFile(),equalTo("/full/path/to/certfile.crt")); - assertThat(httpSinkConfiguration.getSslKeyFile(),equalTo("/full/path/to/keyfile.key")); - assertThat(httpSinkConfiguration.getWorkers(),equalTo(1)); - assertThat(httpSinkConfiguration.getDlqFile(),equalTo("/your/local/dlq-file")); - assertThat(httpSinkConfiguration.getWebhookURL(),equalTo("http://localhost:8080/webhook")); - final Map> customHeaderOptions = httpSinkConfiguration.getCustomHeaderOptions(); - assertThat(customHeaderOptions.get("X-Amzn-SageMaker-Custom-Attributes"),equalTo(List.of("test-attribute"))); - assertThat(customHeaderOptions.get("X-Amzn-SageMaker-Inference-Id"),equalTo(List.of("test-interface-id"))); - assertThat(customHeaderOptions.get("X-Amzn-SageMaker-Enable-Explanations"),equalTo(List.of("test-explanation"))); - assertThat(customHeaderOptions.get("X-Amzn-SageMaker-Target-Variant"),equalTo(List.of("test-target-variant"))); - assertThat(customHeaderOptions.get("X-Amzn-SageMaker-Target-Container-Hostname"),equalTo(List.of("test-container-host"))); - assertThat(customHeaderOptions.get("X-Amzn-SageMaker-Target-Model"),equalTo(List.of("test-target-model"))); + final AwsConfig awsConfig = + httpSinkConfiguration.getAwsConfig(); - final AwsAuthenticationOptions awsAuthenticationOptions = - httpSinkConfiguration.getAwsAuthenticationOptions(); - - assertThat(awsAuthenticationOptions.getAwsRegion(),equalTo(Region.US_EAST_2)); - assertThat(awsAuthenticationOptions.getAwsStsExternalId(),equalTo("test-external-id")); - assertThat(awsAuthenticationOptions.getAwsStsHeaderOverrides().get("test"),equalTo("test")); - assertThat(awsAuthenticationOptions.getAwsStsRoleArn(),equalTo("arn:aws:iam::895099425785:role/data-prepper-s3source-execution-role")); + assertThat(awsConfig.getAwsRegion(),equalTo(Region.US_EAST_2)); + assertThat(awsConfig.getAwsStsExternalId(),equalTo("test-external-id")); + assertThat(awsConfig.getAwsStsHeaderOverrides().get("test"),equalTo("test")); + assertThat(awsConfig.getAwsStsRoleArn(),equalTo("arn:aws:iam::895099425785:role/data-prepper-s3source-execution-role")); final ThresholdOptions thresholdOptions = httpSinkConfiguration.getThresholdOptions(); - assertThat(thresholdOptions.getEventCount(),equalTo(2000)); - assertThat(thresholdOptions.getMaximumSize(),instanceOf(ByteCount.class)); + assertThat(thresholdOptions.getMaxEvents(),equalTo(2000)); + assertThat(thresholdOptions.getMaxRequestSize(),instanceOf(ByteCount.class)); Map pluginSettings = new HashMap<>(); pluginSettings.put("bucket", "dlq.test"); pluginSettings.put("key_path_prefix", "dlq"); final PluginModel pluginModel = new PluginModel("s3", pluginSettings); - assertThat(httpSinkConfiguration.getDlq(), instanceOf(PluginModel.class)); } - @Test - public void validate_and_initialize_cert_and_key_file_in_s3_test() throws JsonProcessingException { - final String SINK_YAML = - " url: \"https://httpbin.org/post\"\n" + - " http_method: \"POST\"\n" + - " auth_type: \"http_basic\"\n" + - " authentication:\n" + - " http_basic:\n" + - " username: \"username\"\n" + - " password: \"vip\"\n" + - " insecure_skip_verify: true\n" + - " use_acm_cert_for_ssl: false\n"+ - " acm_certificate_arn: acm_cert\n" + - " ssl_certificate_file: \"/full/path/to/certfile.crt\"\n" + - " ssl_key_file: \"/full/path/to/keyfile.key\"\n" + - " buffer_type: \"in_memory\"\n" + - " threshold:\n" + - " event_count: 2000\n" + - " maximum_size: 2mb\n" + - " max_retries: 5\n"; - final HttpSinkConfiguration httpSinkConfiguration = objectMapper.readValue(SINK_YAML, HttpSinkConfiguration.class); - httpSinkConfiguration.validateAndInitializeCertAndKeyFileInS3(); - } - - @Test - public void is_valid_aws_url_positive_test() throws JsonProcessingException { - - final String SINK_YAML = - " url: \"https://eihycslfo6g2hwrrytyckjkkok.lambda-url.us-east-2.on.aws/\"\n"; - final HttpSinkConfiguration httpSinkConfiguration = objectMapper.readValue(SINK_YAML, HttpSinkConfiguration.class); - - assertTrue(httpSinkConfiguration.isValidAWSUrl()); - } - - @Test - public void is_valid_aws_url_negative_test() throws JsonProcessingException { - - final String SINK_YAML = - " url: \"http://localhost:8080/post\"\n"; - final HttpSinkConfiguration httpSinkConfiguration = objectMapper.readValue(SINK_YAML, HttpSinkConfiguration.class); - - assertFalse(httpSinkConfiguration.isValidAWSUrl()); - } } diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptionsTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptionsTest.java index a741fa73f9..ac245c08ac 100644 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptionsTest.java +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/configuration/ThresholdOptionsTest.java @@ -1,33 +1,42 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.configuration; import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.Test; import org.opensearch.dataprepper.model.types.ByteCount; +import java.time.Duration; + import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; class ThresholdOptionsTest { private static final String DEFAULT_BYTE_CAPACITY = "50mb"; - private static final int DEFAULT_EVENT_COUNT = 0; + private static final int DEFAULT_EVENT_COUNT = 100; + private static final Duration DEFAULT_EVENT_COLLECT_TIMEOUT = Duration.ofSeconds(10); @Test void default_byte_capacity_test() { - MatcherAssert.assertThat(new ThresholdOptions().getMaximumSize().getBytes(), + MatcherAssert.assertThat(new ThresholdOptions().getMaxRequestSize().getBytes(), equalTo(ByteCount.parse(DEFAULT_BYTE_CAPACITY).getBytes())); } @Test void get_event_collection_duration_test() { - assertThat(new ThresholdOptions().getEventCollectTimeOut(), equalTo(null)); + assertThat(new ThresholdOptions().getFlushTimeOut(), equalTo(DEFAULT_EVENT_COLLECT_TIMEOUT)); } @Test void get_event_count_test() { - assertThat(new ThresholdOptions().getEventCount(), equalTo(DEFAULT_EVENT_COUNT)); + assertThat(new ThresholdOptions().getMaxEvents(), equalTo(DEFAULT_EVENT_COUNT)); } -} \ No newline at end of file +} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/dlq/DlqPushHandlerTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/dlq/DlqPushHandlerTest.java deleted file mode 100644 index d289b78ac4..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/dlq/DlqPushHandlerTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.dlq; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.plugin.PluginFactory; -import org.opensearch.dataprepper.plugins.dlq.DlqProvider; -import org.opensearch.dataprepper.plugins.dlq.DlqWriter; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AwsAuthenticationOptions; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.anyList; -import static org.mockito.Mockito.anyString; - -public class DlqPushHandlerTest { - - private static final String BUCKET = "bucket"; - private static final String BUCKET_VALUE = "test"; - private static final String ROLE = "arn:aws:iam::524239988944:role/app-test"; - - private static final String REGION = "ap-south-1"; - private static final String S3_PLUGIN_NAME = "s3"; - private static final String KEY_PATH_PREFIX = "key_path_prefix"; - - private static final String KEY_PATH_PREFIX_VALUE = "dlq/"; - - private static final String PIPELINE_NAME = "log-pipeline"; - - private static final String DLQ_FILE = "local_dlq_file"; - - private PluginModel pluginModel; - - private DlqPushHandler dlqPushHandler; - private PluginFactory pluginFactory; - - private AwsAuthenticationOptions awsAuthenticationOptions; - - private DlqProvider dlqProvider; - - private DlqWriter dlqWriter; - - @BeforeEach - public void setUp() throws Exception{ - this.pluginFactory = mock(PluginFactory.class); - this.pluginModel = mock(PluginModel.class); - this.awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); - this.dlqProvider = mock(DlqProvider.class); - this.dlqWriter = mock(DlqWriter.class); - } - - @Test - public void perform_for_dlq_s3_success() throws IOException { - Map props = new HashMap<>(); - props.put(BUCKET,BUCKET_VALUE); - props.put(KEY_PATH_PREFIX,KEY_PATH_PREFIX_VALUE); - - when(pluginFactory.loadPlugin(any(Class.class), any(PluginSetting.class))).thenReturn(dlqProvider); - - when(dlqProvider.getDlqWriter(Mockito.anyString())).thenReturn(Optional.of(dlqWriter)); - doNothing().when(dlqWriter).write(anyList(), anyString(), anyString()); - FailedDlqData failedDlqData = FailedDlqData.builder().build(); - dlqPushHandler = new DlqPushHandler(null,pluginFactory, BUCKET_VALUE, ROLE, REGION,KEY_PATH_PREFIX_VALUE); - - PluginSetting pluginSetting = new PluginSetting(S3_PLUGIN_NAME, props); - pluginSetting.setPipelineName(PIPELINE_NAME); - dlqPushHandler.perform(pluginSetting, failedDlqData); - verify(dlqWriter).write(anyList(), anyString(), anyString()); - } - - - @Test - public void perform_for_dlq_local_file_success(){ - - FailedDlqData failedDlqData = FailedDlqData.builder().build(); - dlqPushHandler = new DlqPushHandler(DLQ_FILE,pluginFactory,null, ROLE, REGION,null); - - PluginSetting pluginSetting = new PluginSetting(S3_PLUGIN_NAME, null); - pluginSetting.setPipelineName(PIPELINE_NAME); - dlqPushHandler.perform(pluginSetting, failedDlqData); - } -} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/handler/BasicAuthHttpSinkHandlerTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/handler/BasicAuthHttpSinkHandlerTest.java deleted file mode 100644 index cfe6eb06a0..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/handler/BasicAuthHttpSinkHandlerTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.handler; - -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; -import org.apache.hc.core5.http.HttpHost; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.opensearch.dataprepper.plugins.sink.http.util.HttpSinkUtil; - -import java.net.URL; - -import static org.mockito.ArgumentMatchers.any; - -public class BasicAuthHttpSinkHandlerTest { - - private MockedStatic httpSinkUtilStatic; - - private String urlString = "http://localhost:8080"; - @BeforeEach - public void setUp() throws Exception{ - URL url = new URL(urlString); - httpSinkUtilStatic = Mockito.mockStatic(HttpSinkUtil.class); - httpSinkUtilStatic.when(() -> HttpSinkUtil.getURLByUrlString(any())) - .thenReturn(url); - HttpHost targetHost = new HttpHost(url.toURI().getScheme(), url.getHost(), url.getPort()); - httpSinkUtilStatic.when(() -> HttpSinkUtil.getHttpHostByURL(any(URL.class))) - .thenReturn(targetHost); - } - - @AfterEach - public void tearDown() { - httpSinkUtilStatic.close(); - } - - @Test - public void authenticateTest() { - HttpAuthOptions.Builder httpAuthOptionsBuilder = new HttpAuthOptions.Builder(); - httpAuthOptionsBuilder.setUrl(urlString); - httpAuthOptionsBuilder.setHttpClientBuilder(HttpClients.custom()); - httpAuthOptionsBuilder.setHttpClientConnectionManager(PoolingHttpClientConnectionManagerBuilder.create().build()); - Assertions.assertEquals(urlString, new BasicAuthHttpSinkHandler("test", "test", new PoolingHttpClientConnectionManager()).authenticate(httpAuthOptionsBuilder).getUrl()); - } -} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/handler/BearerTokenAuthHttpSinkHandlerTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/handler/BearerTokenAuthHttpSinkHandlerTest.java deleted file mode 100644 index bf4406ea41..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/handler/BearerTokenAuthHttpSinkHandlerTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.handler; - -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; -import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; -import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.plugins.sink.http.OAuthAccessTokenManager; -import org.opensearch.dataprepper.plugins.sink.http.configuration.BearerTokenOptions; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class BearerTokenAuthHttpSinkHandlerTest { - - private String urlString = "http://localhost:8080"; - - private OAuthAccessTokenManager oAuthAccessTokenManager; - - private BearerTokenOptions bearerTokenOptions; - @BeforeEach - public void setUp() throws Exception{ - bearerTokenOptions = new BearerTokenOptions(); - oAuthAccessTokenManager = mock(OAuthAccessTokenManager.class); - when(oAuthAccessTokenManager.getAccessToken(bearerTokenOptions)).thenReturn("access_token_test"); - } - - @Test - public void authenticateTest() { - - HttpAuthOptions.Builder httpAuthOptionsBuilder = new HttpAuthOptions.Builder(); - httpAuthOptionsBuilder.setUrl(urlString); - httpAuthOptionsBuilder.setHttpClientBuilder(HttpClients.custom()); - httpAuthOptionsBuilder.setHttpClientConnectionManager(PoolingHttpClientConnectionManagerBuilder.create().build()); - httpAuthOptionsBuilder.setClassicHttpRequestBuilder(ClassicRequestBuilder.post()); - Assertions.assertEquals(urlString, new BearerTokenAuthHttpSinkHandler(bearerTokenOptions, new PoolingHttpClientConnectionManager(),oAuthAccessTokenManager).authenticate(httpAuthOptionsBuilder).getUrl()); - } -} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkAwsServiceTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkAwsServiceTest.java deleted file mode 100644 index 4d41c463dc..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkAwsServiceTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.service; - -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AwsAuthenticationOptions; -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.regions.Region; - -import java.io.IOException; -import java.util.HashMap; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class HttpSinkAwsServiceTest { - - private HttpSinkConfiguration httpSinkConfiguration; - - private HttpClientBuilder httpClientBuilder; - - private AwsCredentialsSupplier awsCredentialsSupplier; - - private AwsAuthenticationOptions awsAuthenticationOptions; - - private AwsCredentialsProvider awsCredentialsProvider; - - @BeforeEach - public void setup() throws IOException { - httpSinkConfiguration = mock(HttpSinkConfiguration.class); - httpClientBuilder = mock(HttpClientBuilder.class); - awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); - awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); - awsCredentialsProvider = mock(AwsCredentialsProvider.class); - when(awsAuthenticationOptions.getAwsStsRoleArn()).thenReturn("arn:aws:iam::1234567890:role/app-test"); - when(awsAuthenticationOptions.getAwsStsExternalId()).thenReturn("test"); - when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.of("ap-south-1")); - when(awsAuthenticationOptions.getServiceName()).thenReturn("lambda"); - when(awsAuthenticationOptions.getAwsStsHeaderOverrides()).thenReturn(new HashMap<>()); - when(httpSinkConfiguration.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); - when(awsCredentialsSupplier.getProvider(Mockito.any())).thenReturn(awsCredentialsProvider); - - } - - @Test - public void attachSigV4Test() { - HttpSinkAwsService.attachSigV4(httpSinkConfiguration,httpClientBuilder,awsCredentialsSupplier); - } -} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkServiceTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkServiceTest.java index 1a37476974..a57e5e63f9 100644 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkServiceTest.java +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/HttpSinkServiceTest.java @@ -1,15 +1,19 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.sink.http.service; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import io.micrometer.core.instrument.Counter; -import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; @@ -19,8 +23,11 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.common.sink.DefaultSinkMetrics; +import org.opensearch.dataprepper.common.sink.SinkMetrics; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.codec.OutputCodec; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventHandle; @@ -29,14 +36,12 @@ import org.opensearch.dataprepper.plugins.accumulator.BufferFactory; import org.opensearch.dataprepper.plugins.accumulator.InMemoryBufferFactory; import org.opensearch.dataprepper.plugins.sink.http.FailedHttpResponseInterceptor; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AuthenticationOptions; -import org.opensearch.dataprepper.plugins.sink.http.configuration.AuthTypeOptions; +import org.opensearch.dataprepper.plugins.sink.http.HttpEndpointResponse; +import org.opensearch.dataprepper.plugins.sink.http.HttpSinkSender; import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; import org.opensearch.dataprepper.plugins.sink.http.configuration.ThresholdOptions; -import org.opensearch.dataprepper.plugins.sink.http.dlq.DlqPushHandler; import org.opensearch.dataprepper.test.helper.ReflectivelySetField; -import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -48,7 +53,6 @@ import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -59,37 +63,17 @@ public class HttpSinkServiceTest { private static final String SINK_YAML = " url: \"http://localhost:8080/test\"\n" + - " proxy: \"http://localhost:8080/proxy\"\n" + " codec:\n" + " ndjson:\n" + - " http_method: \"POST\"\n" + - " auth_type: \"unauthenticated\"\n" + - " authentication:\n" + - " http_basic:\n" + - " username: \"username\"\n" + - " password: \"vip\"\n" + - " bearer_token:\n" + - " client_id: 0oaafr4j79segrYGC5d7\n" + - " client_secret: fFel-3FutCXAOndezEsOVlght6D6DR4OIt7G5D1_oJ6w0wNoaYtgU17JdyXmGf0M\n" + - " token_url: https://localhost/oauth2/default/v1/token\n" + - " grant_type: client_credentials\n" + - " scope: httpSink\n"+ - " insecure_skip_verify: true\n" + - " dlq_file: \"/your/local/dlq-file\"\n" + - " dlq:\n" + - " ssl_certificate_file: \"/full/path/to/certfile.crt\"\n" + - " ssl_key_file: \"/full/path/to/keyfile.key\"\n" + - " buffer_type: \"in_memory\"\n" + " aws:\n" + " region: \"us-east-2\"\n" + " sts_role_arn: \"arn:aws:iam::895099425785:role/data-prepper-s3source-execution-role\"\n" + " sts_external_id: \"test-external-id\"\n" + " sts_header_overrides: {\"test\": test }\n" + " threshold:\n" + - " event_count: 1\n" + - " maximum_size: 2mb\n" + - " max_retries: 5\n" + - " aws_sigv4: false\n"; + " max_events: 1\n" + + " max_request_size: 2mb\n" + + " max_retries: 5\n"; private OutputCodec codec; @@ -97,11 +81,10 @@ public class HttpSinkServiceTest { private BufferFactory bufferFactory; - private DlqPushHandler dlqPushHandler; private PluginSetting pluginSetting; - private WebhookService webhookService; + //private WebhookService webhookService; private HttpClientBuilder httpClientBuilder; @@ -117,14 +100,15 @@ public class HttpSinkServiceTest { private CloseableHttpResponse closeableHttpResponse; + private PipelineDescription pipelineDescription; + @BeforeEach void setup() throws Exception { this.codec = mock(OutputCodec.class); this.pluginMetrics = mock(PluginMetrics.class); this.httpSinkConfiguration = objectMapper.readValue(SINK_YAML,HttpSinkConfiguration.class); - this.dlqPushHandler = mock(DlqPushHandler.class); this.pluginSetting = mock(PluginSetting.class); - this.webhookService = mock(WebhookService.class); + //this.webhookService = mock(WebhookService.class); this.httpClientBuilder = mock(HttpClientBuilder.class); this.awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); this.httpSinkRecordsSuccessCounter = mock(Counter.class); @@ -132,27 +116,25 @@ void setup() throws Exception { this.closeableHttpClient = mock(CloseableHttpClient.class); this.closeableHttpResponse = mock(CloseableHttpResponse.class); this.bufferFactory = new InMemoryBufferFactory(); + this.pipelineDescription = mock(PipelineDescription.class); lenient().when(httpClientBuilder.setConnectionManager(Mockito.any())).thenReturn(httpClientBuilder); lenient().when(httpClientBuilder.addResponseInterceptorLast(any(FailedHttpResponseInterceptor.class))).thenReturn(httpClientBuilder); lenient().when(httpClientBuilder.build()).thenReturn(closeableHttpClient); lenient().when(closeableHttpClient.execute(any(ClassicHttpRequest.class),any(HttpClientContext.class))).thenReturn(closeableHttpResponse); - when(pluginMetrics.counter(HttpSinkService.HTTP_SINK_RECORDS_SUCCESS_COUNTER)).thenReturn(httpSinkRecordsSuccessCounter); - when(pluginMetrics.counter(HttpSinkService.HTTP_SINK_RECORDS_FAILED_COUNTER)).thenReturn(httpSinkRecordsFailedCounter); } HttpSinkService createObjectUnderTest(final int eventCount,final HttpSinkConfiguration httpSinkConfig) throws NoSuchFieldException, IllegalAccessException { - ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfig.getThresholdOptions(),"eventCollectTimeOut", Duration.ofNanos(1)); - ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfig.getThresholdOptions(),"eventCount", eventCount); + ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfig.getThresholdOptions(),"flushTimeout", Duration.ofNanos(1)); + ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfig.getThresholdOptions(),"maxEvents", eventCount); + HttpSinkSender httpSender = mock(HttpSinkSender.class); + when(httpSender.send(any(byte[].class))).thenReturn(new HttpEndpointResponse(httpSinkConfig.getUrl(), 200)); + SinkMetrics sinkMetrics = new DefaultSinkMetrics(pluginMetrics, "Event"); return new HttpSinkService( httpSinkConfig, - bufferFactory, - dlqPushHandler, - pluginSetting, - webhookService, - httpClientBuilder, - pluginMetrics, - pluginSetting, + sinkMetrics, + httpSender, + pipelineDescription, codec, null); } @@ -163,7 +145,6 @@ void http_sink_service_test_output_with_single_record() throws NoSuchFieldExcept final Record eventRecord = new Record<>(JacksonEvent.fromMessage("{\"message\":\"c3f847eb-333a-49c3-a4cd-54715ad1b58a\"}")); Collection> records = List.of(eventRecord); objectUnderTest.output(records); - verify(httpSinkRecordsSuccessCounter).increment(1); } @Test @@ -174,47 +155,24 @@ void http_sink_service_test_output_with_multiple_records() throws NoSuchFieldExc for(int record = 0; sinkRecords > record ; record++) records.add(new Record<>(JacksonEvent.fromMessage("{\"message\":" + UUID.randomUUID() + "}"))); objectUnderTest.output(records); - verify(httpSinkRecordsSuccessCounter).increment(sinkRecords); - } - - @Test - void http_sink_service_test_with_internal_server_error() throws NoSuchFieldException, IllegalAccessException, IOException { - final HttpSinkService objectUnderTest = createObjectUnderTest(1,httpSinkConfiguration); - final Record eventRecord = new Record<>(JacksonEvent.fromMessage("{\"message\":\"c3f847eb-333a-49c3-a4cd-54715ad1b58a\"}")); - lenient().when(closeableHttpClient.execute(any(ClassicHttpRequest.class),any(HttpClientContext.class))).thenThrow(new IOException("internal server error")); - objectUnderTest.output(List.of(eventRecord)); - verify(httpSinkRecordsFailedCounter).increment(1); - } - - @Test - void http_sink_service_test_with_single_record_with_basic_authentication() throws NoSuchFieldException, IllegalAccessException, JsonProcessingException { - - final String basicAuthYaml = " http_basic:\n" + - " username: \"username\"\n" + - " password: \"vip\"\n" ; - ReflectivelySetField.setField(HttpSinkConfiguration.class,httpSinkConfiguration,"authentication", objectMapper.readValue(basicAuthYaml, AuthenticationOptions.class)); - ReflectivelySetField.setField(HttpSinkConfiguration.class,httpSinkConfiguration,"authType", AuthTypeOptions.HTTP_BASIC); - final Record eventRecord = new Record<>(JacksonEvent.fromMessage("{\"message\":\"c3f847eb-333a-49c3-a4cd-54715ad1b58a\"}")); - lenient().when(httpClientBuilder.setDefaultCredentialsProvider(any(BasicCredentialsProvider.class))).thenReturn(httpClientBuilder); - final HttpSinkService objectUnderTest = createObjectUnderTest(1,httpSinkConfiguration); - objectUnderTest.output(List.of(eventRecord)); - verify(httpSinkRecordsSuccessCounter).increment(1); } @Test - void http_sink_service_test_with_single_record_with_bearer_token() throws NoSuchFieldException, IllegalAccessException, JsonProcessingException { - lenient().when(httpClientBuilder.setDefaultCredentialsProvider(any(BasicCredentialsProvider.class))).thenReturn(httpClientBuilder); - final String authentication = " bearer_token:\n" + - " client_id: 0oaafr4j79segrYGC5d7\n" + - " client_secret: fFel-3FutCXAOndezEsOVlght6D6DR4OIt7G5D1_oJ6w0wNoaYtgU17JdyXmGf0M\n" + - " token_url: https://localhost/oauth2/default/v1/token\n" + - " grant_type: client_credentials\n" + - " scope: httpSink" ; - ReflectivelySetField.setField(HttpSinkConfiguration.class,httpSinkConfiguration,"authentication", objectMapper.readValue(authentication, AuthenticationOptions.class)); - final HttpSinkService objectUnderTest = createObjectUnderTest(1,httpSinkConfiguration); + void http_sink_service_test_with_internal_server_error() throws NoSuchFieldException, IllegalAccessException { + ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfiguration.getThresholdOptions(),"flushTimeout", Duration.ofNanos(1)); + ReflectivelySetField.setField(ThresholdOptions.class,httpSinkConfiguration.getThresholdOptions(),"maxEvents", 1); + HttpSinkSender httpSender = mock(HttpSinkSender.class); + when(httpSender.send(any(byte[].class))).thenReturn(new HttpEndpointResponse(httpSinkConfiguration.getUrl(), 500, "internal server error")); + SinkMetrics sinkMetrics = new DefaultSinkMetrics(pluginMetrics, "Event"); + final HttpSinkService objectUnderTest = new HttpSinkService( + httpSinkConfiguration, + sinkMetrics, + httpSender, + pipelineDescription, + codec, + null); final Record eventRecord = new Record<>(JacksonEvent.fromMessage("{\"message\":\"c3f847eb-333a-49c3-a4cd-54715ad1b58a\"}")); objectUnderTest.output(List.of(eventRecord)); - verify(httpSinkRecordsSuccessCounter).increment(1); } @Test @@ -234,6 +192,5 @@ void http_sink_service_test_output_with_single_record_ack_release() throws NoSuc given(event.getEventHandle()).willReturn(mock(EventHandle.class)); given(event.jsonBuilder()).willReturn(mock(Event.JsonStringBuilder.class)); objectUnderTest.output(List.of(new Record<>(event))); - verify(httpSinkRecordsSuccessCounter).increment(1); } } diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/WebhookServiceTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/WebhookServiceTest.java deleted file mode 100644 index e42504d7fe..0000000000 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/service/WebhookServiceTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.sink.http.service; - -import io.micrometer.core.instrument.Counter; -import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; -import org.apache.hc.client5.http.protocol.HttpClientContext; -import org.apache.hc.core5.http.ClassicHttpRequest; -import org.apache.hc.core5.http.HttpHost; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.plugins.sink.http.FailedHttpResponseInterceptor; -import org.opensearch.dataprepper.plugins.sink.http.HttpEndPointResponse; - -import org.opensearch.dataprepper.plugins.sink.http.configuration.HttpSinkConfiguration; -import org.opensearch.dataprepper.plugins.sink.http.dlq.FailedDlqData; - -import java.io.IOException; -import java.util.UUID; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class WebhookServiceTest { - - private static final String TEST_URL = "http://localhost:8080/"+ UUID.randomUUID(); - private HttpClientBuilder httpClientBuilder; - private PluginMetrics pluginMetrics; - private HttpSinkConfiguration httpSinkConfiguration; - - private Counter httpSinkWebhookSuccessCounter; - - private Counter httpSinkWebhookFailedCounter; - - private CloseableHttpClient closeableHttpClient; - - private CloseableHttpResponse closeableHttpResponse; - - @BeforeEach - public void setup() throws IOException { - this.httpClientBuilder = mock(HttpClientBuilder.class); - this.pluginMetrics = mock(PluginMetrics.class); - this.httpSinkConfiguration = mock(HttpSinkConfiguration.class); - this.httpSinkWebhookSuccessCounter = mock(Counter.class); - this.httpSinkWebhookFailedCounter = mock(Counter.class); - this.closeableHttpClient = mock(CloseableHttpClient.class); - this.closeableHttpResponse = mock(CloseableHttpResponse.class); - - lenient().when(httpClientBuilder.build()).thenReturn(closeableHttpClient); - lenient().when(httpClientBuilder.addResponseInterceptorLast(any(FailedHttpResponseInterceptor.class))).thenReturn(httpClientBuilder); - lenient().when(httpClientBuilder.setRetryStrategy(any(DefaultHttpRequestRetryStrategy.class))).thenReturn(httpClientBuilder); - - when(pluginMetrics.counter(WebhookService.HTTP_SINK_SUCCESS_WEBHOOKS)).thenReturn(httpSinkWebhookSuccessCounter); - when(pluginMetrics.counter(WebhookService.HTTP_SINK_FAILED_WEBHOOKS)).thenReturn(httpSinkWebhookFailedCounter); - } - - - private WebhookService createObjectUnderTest(){ - return new WebhookService(TEST_URL,httpClientBuilder,pluginMetrics,httpSinkConfiguration); - } - - @Test - public void http_sink_webhook_service_test_with_one_webhook_success_push() throws IOException { - lenient().when(closeableHttpClient.execute(any(ClassicHttpRequest.class),any(HttpClientContext.class))).thenReturn(closeableHttpResponse); - HttpEndPointResponse httpEndPointResponse = new HttpEndPointResponse(TEST_URL,200); - FailedDlqData failedDlqData = - FailedDlqData.builder() - .withUrl(httpEndPointResponse.getUrl()) - .withMessage("Test Data") - .withStatus(httpEndPointResponse.getStatusCode()).build(); - WebhookService webhookService = createObjectUnderTest(); - webhookService.pushWebhook(failedDlqData); - verify(httpSinkWebhookSuccessCounter).increment(); - } - - @Test - public void http_sink_webhook_service_test_with_one_webhook_failed_to_push() throws IOException { - when(closeableHttpClient.execute(any(HttpHost.class),any(ClassicHttpRequest.class),any(HttpClientContext.class))).thenThrow(new IOException("Internal Server Error")); - HttpEndPointResponse httpEndPointResponse = new HttpEndPointResponse(TEST_URL,500); - FailedDlqData failedDlqData = FailedDlqData.builder() - .withMessage("Test Data") - .withStatus(httpEndPointResponse.getStatusCode()) - .withUrl(httpEndPointResponse.getUrl()) - .build(); - WebhookService webhookService = createObjectUnderTest(); - webhookService.pushWebhook(failedDlqData); - verify(httpSinkWebhookFailedCounter).increment(); - } -} diff --git a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtilTest.java b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtilTest.java index 61fda4fb0a..8cffe03ea2 100644 --- a/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtilTest.java +++ b/data-prepper-plugins/http-sink/src/test/java/org/opensearch/dataprepper/plugins/sink/http/util/HttpSinkUtilTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.sink.http.util; import org.apache.hc.core5.http.HttpHost; diff --git a/settings.gradle b/settings.gradle index a5eaa04bf2..4a3c6066aa 100644 --- a/settings.gradle +++ b/settings.gradle @@ -181,7 +181,7 @@ include 'data-prepper-plugins:sqs-source' include 'data-prepper-plugins:sqs-common' include 'data-prepper-plugins:cloudwatch-logs' include 'data-prepper-plugins:sqs-sink' -//include 'data-prepper-plugins:http-sink' +include 'data-prepper-plugins:http-sink' //include 'data-prepper-plugins:sns-sink' include 'data-prepper-plugins:prometheus-sink' include 'data-prepper-plugins:personalize-sink'