diff --git a/data-prepper-plugins/http-common/src/main/java/org/opensearch/dataprepper/plugins/server/CreateServer.java b/data-prepper-plugins/http-common/src/main/java/org/opensearch/dataprepper/plugins/server/CreateServer.java index 7db89822ef..cf5406a61c 100644 --- a/data-prepper-plugins/http-common/src/main/java/org/opensearch/dataprepper/plugins/server/CreateServer.java +++ b/data-prepper-plugins/http-common/src/main/java/org/opensearch/dataprepper/plugins/server/CreateServer.java @@ -5,7 +5,6 @@ package org.opensearch.dataprepper.plugins.server; - import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; import com.linecorp.armeria.common.util.BlockingTaskExecutor; import com.linecorp.armeria.server.HttpService; @@ -40,17 +39,17 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.function.Function; - public class CreateServer { private final ServerConfiguration serverConfiguration; private final Logger LOG; - private final PluginMetrics pluginMetrics; + private final PluginMetrics pluginMetrics; private String sourceName; private String pipelineName; @@ -68,7 +67,53 @@ public CreateServer(final ServerConfiguration serverConfiguration, final Logger this.pipelineName = pipelineName; } - public Server createGRPCServer(final GrpcAuthenticationProvider authenticationProvider, final BindableService grpcService, final CertificateProvider certificateProvider, final MethodDescriptor methodDescriptor) { + /** + * Service configuration class for GRPC services + * @param Request type + * @param Response type + */ + public static class GRPCServiceConfig { + private final BindableService service; + private final String path; + private final MethodDescriptor methodDescriptor; + + public GRPCServiceConfig(BindableService service, String path, MethodDescriptor methodDescriptor) { + this.service = service; + this.path = path; + this.methodDescriptor = methodDescriptor; + } + + public GRPCServiceConfig(BindableService service) { + this.service = service; + this.path = null; + this.methodDescriptor = null; + } + + public BindableService getService() { + return service; + } + + public String getPath() { + return path; + } + + public MethodDescriptor getMethodDescriptor() { + return methodDescriptor; + } + } + + /** + * Creates a GRPC server with multiple services, each with its own optional path and method descriptor + * @param authenticationProvider Provider for authentication + * @param grpcServiceConfigs List of service configurations + * @param certificateProvider Provider for SSL/TLS certificates + * @return Configured server + */ + public Server createGRPCServer( + final GrpcAuthenticationProvider authenticationProvider, + final List> grpcServiceConfigs, + final CertificateProvider certificateProvider) { + final List serverInterceptors = getAuthenticationInterceptor(authenticationProvider); final GrpcServiceBuilder grpcServiceBuilder = GrpcService @@ -77,13 +122,20 @@ public Server createGRPCServer(final GrpcAuthenticationProvider authentic .useBlockingTaskExecutor(true) .exceptionHandler(createGrpExceptionHandler()); - final String sourcePath = serverConfiguration.getPath(); - if (sourcePath != null) { - final String transformedSourcePath = sourcePath.replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); - grpcServiceBuilder.addService(transformedSourcePath, - ServerInterceptors.intercept(grpcService, serverInterceptors), methodDescriptor); - } else { - grpcServiceBuilder.addService(ServerInterceptors.intercept(grpcService, serverInterceptors)); + // Add each service with its own path and method descriptor + for (GRPCServiceConfig serviceConfig : grpcServiceConfigs) { + if (serviceConfig.getPath() != null && serviceConfig.getMethodDescriptor() != null) { + final String transformedPath = serviceConfig.getPath().replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); + grpcServiceBuilder.addService( + transformedPath, + ServerInterceptors.intercept(serviceConfig.getService(), serverInterceptors), + serviceConfig.getMethodDescriptor()); + LOG.info("Adding service with path: {}", transformedPath); + } else { + grpcServiceBuilder.addService( + ServerInterceptors.intercept(serviceConfig.getService(), serverInterceptors)); + LOG.info("Adding service without specific path"); + } } if (serverConfiguration.hasHealthCheck()) { @@ -151,7 +203,35 @@ public Server createGRPCServer(final GrpcAuthenticationProvider authentic return sb.build(); } - public Server createHTTPServer(final Buffer> buffer, final CertificateProviderFactory certificateProviderFactory, final ArmeriaHttpAuthenticationProvider authenticationProvider, final HttpRequestExceptionHandler httpRequestExceptionHandler, final Object logService) { + /** + * Creates a GRPC server with a single service + * @param authenticationProvider Provider for authentication + * @param grpcService Service to be added + * @param certificateProvider Provider for SSL/TLS certificates + * @param methodDescriptor Method descriptor for the service + * @return Configured server + */ + public Server createGRPCServer( + final GrpcAuthenticationProvider authenticationProvider, + final BindableService grpcService, + final CertificateProvider certificateProvider, + final MethodDescriptor methodDescriptor) { + + List> serviceConfigs = new ArrayList<>(); + if (serverConfiguration.getPath() != null) { + serviceConfigs.add(new GRPCServiceConfig<>(grpcService, serverConfiguration.getPath(), methodDescriptor)); + } else { + serviceConfigs.add(new GRPCServiceConfig<>(grpcService)); + } + + return createGRPCServer(authenticationProvider, serviceConfigs, certificateProvider); + } + + + public Server createHTTPServer(final Buffer> buffer, + final CertificateProviderFactory certificateProviderFactory, + final ArmeriaHttpAuthenticationProvider authenticationProvider, + final HttpRequestExceptionHandler httpRequestExceptionHandler, final Object logService) { final ServerBuilder sb = Server.builder(); sb.disableServerHeader(); @@ -163,17 +243,16 @@ public Server createHTTPServer(final Buffer> buffer, final Certifica // TODO: enable encrypted key with password sb.https(serverConfiguration.getPort()).tls( new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) - ) - ); + new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8))); } else { LOG.warn("Creating " + sourceName + " without SSL/TLS. This is not secure."); LOG.warn("In order to set up TLS for the " + sourceName + ", go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/http-source#ssl"); sb.http(serverConfiguration.getPort()); } - if(serverConfiguration.getAuthentication() != null) { - final Optional> optionalAuthDecorator = authenticationProvider.getAuthenticationDecorator(); + if (serverConfiguration.getAuthentication() != null) { + final Optional> optionalAuthDecorator = authenticationProvider + .getAuthenticationDecorator(); if (serverConfiguration.isUnauthenticatedHealthCheck()) { optionalAuthDecorator.ifPresent(authDecorator -> sb.decorator(REGEX_HEALTH, authDecorator)); @@ -184,7 +263,7 @@ public Server createHTTPServer(final Buffer> buffer, final Certifica sb.maxNumConnections(serverConfiguration.getMaxConnectionCount()); sb.requestTimeout(Duration.ofMillis(serverConfiguration.getRequestTimeoutInMillis())); - if(serverConfiguration.getMaxRequestLength() != null) { + if (serverConfiguration.getMaxRequestLength() != null) { sb.maxRequestLength(serverConfiguration.getMaxRequestLength().getBytes()); } final int threads = serverConfiguration.getThreadCount(); @@ -193,7 +272,8 @@ public Server createHTTPServer(final Buffer> buffer, final Certifica final int maxPendingRequests = serverConfiguration.getMaxPendingRequests(); final LogThrottlingStrategy logThrottlingStrategy = new LogThrottlingStrategy( maxPendingRequests, blockingTaskExecutor.getQueue()); - final LogThrottlingRejectHandler logThrottlingRejectHandler = new LogThrottlingRejectHandler(maxPendingRequests, pluginMetrics); + final LogThrottlingRejectHandler logThrottlingRejectHandler = new LogThrottlingRejectHandler(maxPendingRequests, + pluginMetrics); final String httpSourcePath = serverConfiguration.getPath().replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); sb.decorator(httpSourcePath, ThrottlingService.newDecorator(logThrottlingStrategy, logThrottlingRejectHandler)); @@ -201,7 +281,8 @@ public Server createHTTPServer(final Buffer> buffer, final Certifica if (CompressionOption.NONE.equals(serverConfiguration.getCompression())) { sb.annotatedService(httpSourcePath, logService, httpRequestExceptionHandler); } else { - sb.annotatedService(httpSourcePath, logService, DecodingService.newDecorator(), httpRequestExceptionHandler); + sb.annotatedService(httpSourcePath, logService, DecodingService.newDecorator(), + httpRequestExceptionHandler); } if (serverConfiguration.hasHealthCheck()) { @@ -220,12 +301,12 @@ private GrpcExceptionHandlerFunction createGrpExceptionHandler() { return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); } - private List getAuthenticationInterceptor(final GrpcAuthenticationProvider authenticationProvider) { + private List getAuthenticationInterceptor( + final GrpcAuthenticationProvider authenticationProvider) { final ServerInterceptor authenticationInterceptor = authenticationProvider.getAuthenticationInterceptor(); if (authenticationInterceptor == null) { return Collections.emptyList(); } return Collections.singletonList(authenticationInterceptor); } -} -// \ No newline at end of file +} \ No newline at end of file diff --git a/data-prepper-plugins/http-common/src/test/java/org/opensearch/dataprepper/plugins/server/CreateServerTest.java b/data-prepper-plugins/http-common/src/test/java/org/opensearch/dataprepper/plugins/server/CreateServerTest.java index eaaae6a0df..25609d2366 100644 --- a/data-prepper-plugins/http-common/src/test/java/org/opensearch/dataprepper/plugins/server/CreateServerTest.java +++ b/data-prepper-plugins/http-common/src/test/java/org/opensearch/dataprepper/plugins/server/CreateServerTest.java @@ -8,7 +8,10 @@ import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServiceRequestContext; + import io.grpc.ServerInterceptor; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -27,6 +30,7 @@ import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoStandardCodec; +import org.opensearch.dataprepper.plugins.server.CreateServer.GRPCServiceConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +38,10 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -51,7 +57,8 @@ public class CreateServerTest { private Logger LOG = LoggerFactory.getLogger(CreateServer.class); ObjectMapper objectMapper; private final String TEST_SSL_CERTIFICATE_FILE = getClass().getClassLoader().getResource("test_cert.crt").getFile(); - private final String TEST_SSL_KEY_FILE = getClass().getClassLoader().getResource("test_decrypted_key.key").getFile(); + private final String TEST_SSL_KEY_FILE = getClass().getClassLoader().getResource("test_decrypted_key.key") + .getFile(); private String TEST_PIPELINE_NAME = "test-pipeline"; private String TEST_SOURCE_NAME = "test-source"; @@ -82,9 +89,11 @@ public class CreateServerTest { @Test void createGrpcServerTest() throws JsonProcessingException { when(authenticationProvider.getAuthenticationInterceptor()).thenReturn(authenticationInterceptor); - final Map metadata = createGrpcMetadata(21890, false, 10000, 10, 5, CompressionOption.NONE, null); + final Map metadata = createGrpcMetadata(21890, false, 10000, 10, 5, CompressionOption.NONE, + null); final ServerConfiguration serverConfiguration = createServerConfig(metadata); - final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, TEST_PIPELINE_NAME); + final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, + TEST_PIPELINE_NAME); Buffer> buffer = new BlockingBuffer>(TEST_PIPELINE_NAME); TestService testService = getTestService(buffer); @@ -96,9 +105,37 @@ void createGrpcServerTest() throws JsonProcessingException { } @Test - void testCustomAuthModule() throws JsonProcessingException{ + void createGrpcServerTestWithServiceConfigs() throws JsonProcessingException { when(authenticationProvider.getAuthenticationInterceptor()).thenReturn(authenticationInterceptor); - final Map metadata = createGrpcMetadata(21890, false, 10000, 10, 5, CompressionOption.NONE, null); + final Map metadata = createGrpcMetadata(21890, false, 10000, 10, 5, CompressionOption.NONE, + null); + final ServerConfiguration serverConfiguration = createServerConfig(metadata); + final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, + TEST_PIPELINE_NAME); + Buffer> buffer = new BlockingBuffer>(TEST_PIPELINE_NAME); + + // Create test services + TestService testService1 = getTestService(buffer); + TestService testService2 = getTestService(buffer); + TestService testService3 = getTestService(buffer); + + List> serviceConfigs = new ArrayList<>(); + serviceConfigs.add(new GRPCServiceConfig<>(testService1, "/v1/metrics", MetricsServiceGrpc.getExportMethod())); + serviceConfigs.add(new GRPCServiceConfig<>(testService2, "/v2/metrics", MetricsServiceGrpc.getExportMethod())); + serviceConfigs.add(new GRPCServiceConfig<>(testService3, null, null)); + + Server server = createServer.createGRPCServer(authenticationProvider, serviceConfigs, certificateProvider); + + assertNotNull(server); + assertDoesNotThrow(() -> server.start()); + assertDoesNotThrow(() -> server.stop()); + } + + @Test + void testCustomAuthModule() throws JsonProcessingException { + when(authenticationProvider.getAuthenticationInterceptor()).thenReturn(authenticationInterceptor); + final Map metadata = createGrpcMetadata(21890, false, 10000, 10, 5, CompressionOption.NONE, + null); Map authConfig = new HashMap<>(); authConfig.put("plugin", "test-auth"); @@ -111,7 +148,8 @@ void testCustomAuthModule() throws JsonProcessingException{ customAuth.setLogger(mockLogger); when(authenticationProvider.getHttpAuthenticationService()).thenReturn(Optional.of(customAuth)); - final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, TEST_PIPELINE_NAME); + final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, + TEST_PIPELINE_NAME); Buffer> buffer = new BlockingBuffer>(TEST_PIPELINE_NAME); TestService testService = getTestService(buffer); Server server = createServer.createGRPCServer(authenticationProvider, testService, certificateProvider, null); @@ -120,8 +158,7 @@ void testCustomAuthModule() throws JsonProcessingException{ assertDoesNotThrow(() -> server.start()); WebClient webClient = WebClient.builder( - String.format("http://127.0.0.1:%d", serverConfiguration.getPort()) - ).build(); + String.format("http://127.0.0.1:%d", serverConfiguration.getPort())).build(); webClient.get("/").aggregate().join(); verify(mockLogger).info("Ensure Custom Auth Decorator is working"); @@ -140,20 +177,22 @@ void createHttpServerTest() throws IOException { when(certificate.getPrivateKey()).thenReturn(keyAsString); when(certificateProvider.getCertificate()).thenReturn(certificate); when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map metadata = createHttpMetadata(2021, "/log/ingest", 10_000, 200, 500, 1024, true, CompressionOption.NONE); + final Map metadata = createHttpMetadata(2021, "/log/ingest", 10_000, 200, 500, 1024, true, + CompressionOption.NONE); final ServerConfiguration serverConfiguration = createServerConfig(metadata); - final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, TEST_PIPELINE_NAME); + final CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, TEST_SOURCE_NAME, + TEST_PIPELINE_NAME); Buffer> buffer = new BlockingBuffer>(TEST_PIPELINE_NAME); String logService = "placeholder"; - Server server = createServer.createHTTPServer(buffer, certificateProviderFactory, armeriaAuthenticationProvider, httpRequestExceptionHandler, logService); + Server server = createServer.createHTTPServer(buffer, certificateProviderFactory, armeriaAuthenticationProvider, + httpRequestExceptionHandler, logService); assertNotNull(server); assertDoesNotThrow(() -> server.start()); assertDoesNotThrow(() -> server.stop()); } - - - private Map createGrpcMetadata (Integer port, Boolean ssl, Integer reqeustTimeoutInMillis, Integer maxConnectionCount, Integer threadCount, CompressionOption compression, RetryInfoConfig retryInfo){ + private Map createGrpcMetadata(Integer port, Boolean ssl, Integer reqeustTimeoutInMillis, + Integer maxConnectionCount, Integer threadCount, CompressionOption compression, RetryInfoConfig retryInfo) { final Map metadata = new HashMap<>(); metadata.put("port", port); metadata.put("ssl", ssl); @@ -165,7 +204,9 @@ private Map createGrpcMetadata (Integer port, Boolean ssl, Integ return metadata; } - private Map createHttpMetadata (Integer port, String path, Integer requestTimeoutInMillis, Integer threadCount, Integer maxConnectionCount, Integer maxPendingRequests, Boolean hasHealthCheckService, CompressionOption compressionOption){ + private Map createHttpMetadata(Integer port, String path, Integer requestTimeoutInMillis, + Integer threadCount, Integer maxConnectionCount, Integer maxPendingRequests, Boolean hasHealthCheckService, + CompressionOption compressionOption) { final Map metadata = new HashMap<>(); metadata.put("port", port); metadata.put("path", path); @@ -185,13 +226,12 @@ private ServerConfiguration createServerConfig(final Map metadat return objectMapper.readValue(json, ServerConfiguration.class); } - private TestService getTestService(Buffer> buffer){ + private TestService getTestService(Buffer> buffer) { TestService testService = new TestService( 80, new OTelProtoStandardCodec.OTelProtoDecoder(), buffer, - pluginMetrics - ); + pluginMetrics); return testService; } @@ -206,7 +246,8 @@ void setLogger(Logger logger) { public HttpService apply(HttpService delegate) { return new HttpService() { @Override - public com.linecorp.armeria.common.HttpResponse serve(ServiceRequestContext ctx, HttpRequest request) throws Exception { + public com.linecorp.armeria.common.HttpResponse serve(ServiceRequestContext ctx, HttpRequest request) + throws Exception { authLog.info("Ensure Custom Auth Decorator is working"); return delegate.serve(ctx, request); } diff --git a/data-prepper-plugins/otlp-source/README.md b/data-prepper-plugins/otlp-source/README.md new file mode 100644 index 0000000000..6e9611ccf3 --- /dev/null +++ b/data-prepper-plugins/otlp-source/README.md @@ -0,0 +1,137 @@ +# OTLP Source + +This is a source which follows the [OTLP Protocol](https://opentelemetry.io/docs/specs/otlp/) and supports three endpoints for logs, metrics, and traces. It supports both `OTLP/grpc` and `OTLP/HTTP`. + +## Usages + +### Routing telemetry signal based on event type + +Each of the telemetry signals may be sent to different processors or sinks based on specific needs. The routing to downstream pipelines is based on meta-data routing. + +```yaml +otel-telemetry-pipeline: + source: + otlp: + ssl: false + route: + - logs: 'getEventType() == "LOG"' + - traces: 'getEventType() == "TRACE"' + - metrics: 'getEventType() == "METRIC"' + sink: + - pipeline: + name: "logs-pipeline" + routes: + - "logs" + - pipeline: + name: "traces-pipeline" + routes: + - "traces" + - pipeline: + name: "metrics-pipeline" + routes: + - "metrics" +``` + +## Configurations + +- port(Optional) => An `int` represents the port OTLP source is running on. Default is `21893`. +- logs_path(Optional) => A `String` which represents the path for sending unframed HTTP requests for logs. It should start with `/` and length should be at least 1. Default is `/opentelemetry.proto.collector.logs.v1.LogsService/Export`. +- metrics_path(Optional) => A `String` which represents the path for sending unframed HTTP requests for metrics. It should start with `/` and length should be at least 1. Default is `/opentelemetry.proto.collector.metrics.v1.MetricsService/Export`. +- traces_path(Optional) => A `String` which represents the path for sending unframed HTTP requests for traces. It should start with `/` and length should be at least 1. Default is `/opentelemetry.proto.collector.trace.v1.TraceService/Export`. +- request_timeout(Optional) => An `int` represents request timeout in millis. Default is `10000`. +- health_check_service(Optional) => A boolean enables a gRPC health check service under `grpc.health.v1.Health/Check`. Default is `false`. +- proto_reflection_service(Optional) => A boolean enables a reflection service for Protobuf services (see [ProtoReflectionService](https://grpc.github.io/grpc-java/javadoc/io/grpc/protobuf/services/ProtoReflectionService.html) and [gRPC reflection](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is `false`. +- unframed_requests(Optional) => A boolean to enable requests not framed using the gRPC wire protocol. When `health_check_service` is true and `unframed_requests` is true, enables HTTP health check service under `/health`. +- thread_count(Optional) => the number of threads to keep in the ScheduledThreadPool. Default is `200`. +- max_connection_count(Optional) => the maximum allowed number of open connections. Default is `500`. +- compression(Optional) => The compression type applied on the client request payload. Defaults to `none`. Supported values are: + - `none`: no compression + - `gzip`: apply GZip de-compression on the incoming request. +- output_format(Optional) => Specifies the decoded output format for all signals (logs, metrics, traces) if individual output format options are not set. Supported values are: + - `otel`: OpenTelemetry format (default). + - `opensearch`: OpenSearch format. +- logs_output_format(Optional) => Specifies the decoded output format for logs. Supported values are: + - `otel`: OpenTelemetry format (default). + - `opensearch`: OpenSearch format. +- metrics_output_format(Optional) => Specifies the decoded output format for metrics. Supported values are: + - `otel`: OpenTelemetry format (default). + - `opensearch`: OpenSearch format. +- traces_output_format(Optional) => Specifies the decoded output format for traces. Supported values are: + - `otel`: OpenTelemetry format (default). + - `opensearch`: OpenSearch format. + +> **Note:** If an individual output format (e.g., `logs_output_format`) is set, it takes precedence over the generic `output_format` for that signal type. If neither is set, the default is `otel`. + +### Retry Information + +Data Prepper replies with a `RetryInfo` specifying how long to wait for the next request in case backpressure builds up. The retry information is implemented as exponential backoff, with a max delay of `retry_info.max_delay`. + +```yaml +source: + otlp: + retry_info: + min_delay: 1000ms # defaults to 100ms + max_delay: 5s # defaults to 2s +``` + +### Authentication Configurations + +By default, the otlp input is unauthenticated. + +The following is an example of how to run the server with HTTP Basic authentication: + +```yaml +source: + otlp: + authentication: + http_basic: + username: my-user + password: my_s3cr3t +``` + +You can also explicitly disable authentication with: + +```yaml +source: + otlp: + authentication: + unauthenticated: +``` + +This plugin uses pluggable authentication for GRPC servers. To provide custom authentication, +create a plugin which implements [`GrpcAuthenticationProvider`](../armeria-common/src/main/java/org/opensearch/dataprepper/armeria/authentication/GrpcAuthenticationProvider.java) + +### SSL + +- ssl(Optional) => A boolean enables TLS/SSL. Default is `true`. +- sslKeyCertChainFile(Optional) => A `String` represents the SSL certificate chain file path or AWS S3 path. S3 path example `s3:///`. Required if `ssl` is set to `true`. +- sslKeyFile(Optional) => A `String` represents the SSL key file path or AWS S3 path. S3 path example `s3:///`. Required if `ssl` is set to `true`. +- useAcmCertForSSL(Optional) => A boolean enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is `false`. +- acmCertificateArn(Optional) => A `String` represents the ACM certificate ARN. ACM certificate takes preference over S3 or local file system certificate. Required if `useAcmCertForSSL` is set to `true`. +- awsRegion(Optional) => A `String` represents the AWS region to use ACM or S3. Required if `useAcmCertForSSL` is set to `true` or `sslKeyCertChainFile` and `sslKeyFile` is `AWS S3 path`. + +## Metrics + +### Counter + +- `requestTimeouts`: measures total number of requests that time out. +- `requestsReceived`: measures total number of requests received by OTLP source. +- `successRequests`: measures total number of requests successfully processed by OTLP source plugin. +- `badRequests`: measures total number of requests with invalid format processed by OTLP source plugin. +- `requestsTooLarge`: measures total number of requests that exceed the maximum allowed size. +- `internalServerError`: measures total number of requests processed by OTLP source with custom exception type. + +### Timer + +- `requestProcessDuration`: measures latency of requests processed by OTLP source plugin in seconds. + +### Distribution Summary + +- `payloadSize`: measures the distribution of incoming requests payload sizes in bytes. + +## Developer Guide + +This plugin is compatible with Java 8. See + +- [CONTRIBUTING](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md) +- [monitoring](https://github.com/opensearch-project/data-prepper/blob/main/docs/monitoring.md) diff --git a/data-prepper-plugins/otlp-source/build.gradle b/data-prepper-plugins/otlp-source/build.gradle new file mode 100644 index 0000000000..2d275a63c6 --- /dev/null +++ b/data-prepper-plugins/otlp-source/build.gradle @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java' +} + +dependencies { + implementation project(':data-prepper-api') + implementation project(':data-prepper-plugins:common') + implementation project(':data-prepper-plugins:blocking-buffer') + implementation project(':data-prepper-plugins:otel-proto-common') + implementation project(':data-prepper-plugins:otel-logs-source') + implementation project(':data-prepper-plugins:otel-metrics-source') + implementation project(':data-prepper-plugins:otel-trace-source') + implementation project(':data-prepper-plugins:http-common') + implementation libs.commons.codec + implementation project(':data-prepper-plugins:armeria-common') + implementation libs.opentelemetry.proto + implementation libs.commons.io + implementation 'software.amazon.awssdk:acm' + implementation 'software.amazon.awssdk:auth' + implementation 'software.amazon.awssdk:regions' + implementation 'software.amazon.awssdk:s3' + implementation libs.protobuf.util + implementation libs.armeria.core + implementation libs.armeria.grpc + implementation libs.grpc.inprocess + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation libs.commons.lang3 + implementation libs.bouncycastle.bcprov + implementation libs.bouncycastle.bcpkix + implementation 'org.hibernate.validator:hibernate-validator:8.0.0.Final' + testImplementation project(':data-prepper-api').sourceSets.test.output +} + +jacocoTestCoverageVerification { + dependsOn jacocoTestReport + violationRules { + rule { //in addition to core projects rule + limit { + minimum = 0.90 + } + } + } +} \ No newline at end of file diff --git a/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/ConvertConfiguration.java b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/ConvertConfiguration.java new file mode 100644 index 0000000000..252213e015 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/ConvertConfiguration.java @@ -0,0 +1,26 @@ +package org.opensearch.dataprepper.plugins.source.otlp; + +import org.opensearch.dataprepper.plugins.server.ServerConfiguration; + +public class ConvertConfiguration { + + public static ServerConfiguration convertConfiguration(final OTLPSourceConfig otlpSourceConfig) { + ServerConfiguration serverConfiguration = new ServerConfiguration(); + serverConfiguration.setHealthCheck(otlpSourceConfig.hasHealthCheck()); + serverConfiguration.setProtoReflectionService(otlpSourceConfig.hasProtoReflectionService()); + serverConfiguration.setRequestTimeoutInMillis(otlpSourceConfig.getRequestTimeoutInMillis()); + serverConfiguration.setEnableUnframedRequests(otlpSourceConfig.enableUnframedRequests()); + serverConfiguration.setCompression(otlpSourceConfig.getCompression()); + serverConfiguration.setAuthentication(otlpSourceConfig.getAuthentication()); + serverConfiguration.setSsl(otlpSourceConfig.isSsl()); + serverConfiguration.setUnauthenticatedHealthCheck(otlpSourceConfig.isUnauthenticatedHealthCheck()); + serverConfiguration.setUseAcmCertForSSL(otlpSourceConfig.useAcmCertForSSL()); + serverConfiguration.setMaxRequestLength(otlpSourceConfig.getMaxRequestLength()); + serverConfiguration.setPort(otlpSourceConfig.getPort()); + serverConfiguration.setRetryInfo(otlpSourceConfig.getRetryInfo()); + serverConfiguration.setThreadCount(otlpSourceConfig.getThreadCount()); + serverConfiguration.setMaxConnectionCount(otlpSourceConfig.getMaxConnectionCount()); + + return serverConfiguration; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSource.java b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSource.java new file mode 100644 index 0000000000..51828f63e6 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSource.java @@ -0,0 +1,180 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otlp; + +import com.linecorp.armeria.server.Server; + +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; + +import org.opensearch.dataprepper.model.source.Source; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.otel.codec.OTelOutputFormat; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoOpensearchCodec; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoStandardCodec; +import org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsGrpcService; +import org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsGrpcService; +import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; +import org.opensearch.dataprepper.plugins.source.otlp.certificate.CertificateProviderFactory; +import org.opensearch.dataprepper.plugins.server.CreateServer; +import org.opensearch.dataprepper.plugins.server.CreateServer.GRPCServiceConfig; +import org.opensearch.dataprepper.plugins.server.ServerConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.metric.Metric; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.PluginMetrics; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; + +@DataPrepperPlugin(name = "otlp", pluginType = Source.class, pluginConfigurationType = OTLPSourceConfig.class) +public class OTLPSource implements Source> { + private static final Logger LOG = LoggerFactory.getLogger(OTLPSource.class); + static final String SERVER_CONNECTIONS = "serverConnections"; + + private final OTLPSourceConfig otlpSourceConfig; + private final String pipelineName; + private final PluginMetrics pluginMetrics; + private final GrpcAuthenticationProvider authenticationProvider; + private final CertificateProviderFactory certificateProviderFactory; + private Server server; + + @DataPrepperPluginConstructor + public OTLPSource(final OTLPSourceConfig otlpSourceConfig, + final PluginMetrics pluginMetrics, + final PluginFactory pluginFactory, + final PipelineDescription pipelineDescription) { + this(otlpSourceConfig, pluginMetrics, pluginFactory, + new CertificateProviderFactory(otlpSourceConfig), pipelineDescription); + } + + // accessible only in the same package for unit test + OTLPSource(final OTLPSourceConfig otlpSourceConfig, + final PluginMetrics pluginMetrics, + final PluginFactory pluginFactory, + final CertificateProviderFactory certificateProviderFactory, + final PipelineDescription pipelineDescription) { + otlpSourceConfig.validateAndInitializeCertAndKeyFileInS3(); + this.otlpSourceConfig = otlpSourceConfig; + this.pluginMetrics = pluginMetrics; + this.certificateProviderFactory = certificateProviderFactory; + this.pipelineName = pipelineDescription.getPipelineName(); + this.authenticationProvider = createAuthenticationProvider(pluginFactory); + } + + @Override + public void start(Buffer> buffer) { + if (buffer == null) { + throw new IllegalStateException("Buffer provided is null"); + } + + if (server == null) { + @SuppressWarnings("unchecked") + Buffer> metricBuffer = (Buffer>) (Object) buffer; + + final OTelLogsGrpcService oTelLogsGrpcService = new OTelLogsGrpcService( + (int) (otlpSourceConfig.getRequestTimeoutInMillis() * 0.8), + otlpSourceConfig.getLogsOutputFormat() == OTelOutputFormat.OPENSEARCH ? new OTelProtoOpensearchCodec.OTelProtoDecoder() : new OTelProtoStandardCodec.OTelProtoDecoder(), + buffer, pluginMetrics); + + final OTelMetricsGrpcService oTelMetricsGrpcService = new OTelMetricsGrpcService( + (int) (otlpSourceConfig.getRequestTimeoutInMillis() * 0.8), + otlpSourceConfig.getMetricsOutputFormat() == OTelOutputFormat.OPENSEARCH ? new OTelProtoOpensearchCodec.OTelProtoDecoder() : new OTelProtoStandardCodec.OTelProtoDecoder(), + metricBuffer, pluginMetrics); + + final OTelTraceGrpcService oTelTraceGrpcService = new OTelTraceGrpcService( + (int) (otlpSourceConfig.getRequestTimeoutInMillis() * 0.8), + otlpSourceConfig.getTracesOutputFormat() == OTelOutputFormat.OPENSEARCH ? new OTelProtoOpensearchCodec.OTelProtoDecoder() : new OTelProtoStandardCodec.OTelProtoDecoder(), + buffer, pluginMetrics); + + ServerConfiguration serverConfiguration = ConvertConfiguration.convertConfiguration(otlpSourceConfig); + CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, "otlp", pipelineName); + CertificateProvider certificateProvider = null; + if (otlpSourceConfig.isSsl() || otlpSourceConfig.useAcmCertForSSL()) { + certificateProvider = certificateProviderFactory.getCertificateProvider(); + } + + List> serviceConfigs = new ArrayList<>(); + serviceConfigs.add(new GRPCServiceConfig<>(oTelLogsGrpcService, otlpSourceConfig.getLogsPath(), + LogsServiceGrpc.getExportMethod())); + serviceConfigs.add(new GRPCServiceConfig<>(oTelMetricsGrpcService, otlpSourceConfig.getMetricsPath(), + MetricsServiceGrpc.getExportMethod())); + serviceConfigs.add(new GRPCServiceConfig<>(oTelTraceGrpcService, otlpSourceConfig.getTracesPath(), + TraceServiceGrpc.getExportMethod())); + + server = createServer.createGRPCServer(authenticationProvider, serviceConfigs, certificateProvider); + + pluginMetrics.gauge(SERVER_CONNECTIONS, server, Server::numConnections); + } + + try { + server.start().get(); + } catch (ExecutionException ex) { + if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { + throw (RuntimeException) ex.getCause(); + } else { + throw new RuntimeException(ex); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new RuntimeException(ex); + } + LOG.info("Started otlp source on port {}.", otlpSourceConfig.getPort()); + } + + @Override + public void stop() { + if (server != null) { + try { + server.stop().get(); + } catch (ExecutionException ex) { + if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { + throw (RuntimeException) ex.getCause(); + } else { + throw new RuntimeException(ex); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new RuntimeException(ex); + } + } + LOG.info("Stopped otlp source."); + } + + private GrpcAuthenticationProvider createAuthenticationProvider(final PluginFactory pluginFactory) { + final PluginModel authenticationConfiguration = otlpSourceConfig.getAuthentication(); + + if (authenticationConfiguration == null || authenticationConfiguration.getPluginName() + .equals(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { + LOG.warn("Creating otlp source without authentication. This is not secure."); + LOG.warn( + "In order to set up Http Basic authentication for otlp source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otlp-source#authentication-configurations"); + } + + final PluginSetting authenticationPluginSetting; + if (authenticationConfiguration != null) { + authenticationPluginSetting = new PluginSetting(authenticationConfiguration.getPluginName(), + authenticationConfiguration.getPluginSettings()); + } else { + authenticationPluginSetting = new PluginSetting(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, + Collections.emptyMap()); + } + authenticationPluginSetting.setPipelineName(pipelineName); + return pluginFactory.loadPlugin(GrpcAuthenticationProvider.class, authenticationPluginSetting); + } +} diff --git a/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceConfig.java b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceConfig.java new file mode 100644 index 0000000000..1f1b5eaf11 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceConfig.java @@ -0,0 +1,324 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otlp; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Size; +import org.apache.commons.lang3.StringUtils; +import org.opensearch.dataprepper.model.types.ByteCount; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.otel.codec.OTelOutputFormat; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.hibernate.validator.constraints.time.DurationMax; +import org.hibernate.validator.constraints.time.DurationMin; + +import java.time.Duration; + +public class OTLPSourceConfig { + static final String REQUEST_TIMEOUT = "request_timeout"; + static final String PORT = "port"; + static final String LOGS_PATH = "logs_path"; + static final String METRICS_PATH = "metrics_path"; + static final String TRACES_PATH = "traces_path"; + static final String SSL = "ssl"; + static final String OUTPUT_FORMAT = "output_format"; + static final String LOGS_OUTPUT_FORMAT = "logs_output_format"; + static final String METRICS_OUTPUT_FORMAT = "metrics_output_format"; + static final String TRACES_OUTPUT_FORMAT = "traces_output_format"; + static final String USE_ACM_CERT_FOR_SSL = "use_acm_certificate_for_ssl"; + static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acm_certificate_timeout"; + static final String HEALTH_CHECK_SERVICE = "health_check_service"; + static final String PROTO_REFLECTION_SERVICE = "proto_reflection_service"; + static final String SSL_KEY_CERT_FILE = "ssl_certificate_file"; + static final String SSL_KEY_FILE = "ssl_key_file"; + static final String ACM_CERT_ARN = "acm_certificate_arn"; + static final String ACM_PRIVATE_KEY_PASSWORD = "acm_private_key_password"; + static final String AWS_REGION = "aws_region"; + static final String THREAD_COUNT = "thread_count"; + static final String MAX_CONNECTION_COUNT = "max_connection_count"; + static final String ENABLE_UNFRAMED_REQUESTS = "unframed_requests"; + static final String COMPRESSION = "compression"; + static final String RETRY_INFO = "retry_info"; + static final int DEFAULT_REQUEST_TIMEOUT = 10; // in seconds + static final int DEFAULT_PORT = 21893; + static final int DEFAULT_THREAD_COUNT = 200; + static final int DEFAULT_MAX_CONNECTION_COUNT = 500; + static final boolean DEFAULT_SSL = true; + static final boolean DEFAULT_ENABLED_UNFRAMED_REQUESTS = false; + static final boolean DEFAULT_HEALTH_CHECK = false; + static final boolean DEFAULT_PROTO_REFLECTION_SERVICE = false; + static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false; + static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT = 120; // in seconds + private static final String S3_PREFIX = "s3://"; + static final String UNAUTHENTICATED_HEALTH_CHECK = "unauthenticated_health_check"; + + @JsonProperty(REQUEST_TIMEOUT) + @DurationMin(seconds = 5) + @DurationMax(seconds = 3600) + private Duration requestTimeout = Duration.ofSeconds(DEFAULT_REQUEST_TIMEOUT); + + @JsonProperty(PORT) + private int port = DEFAULT_PORT; + + @JsonProperty(LOGS_PATH) + @Size(min = 1, message = "logsPath length should be at least 1") + private String logsPath; + + @JsonProperty(METRICS_PATH) + @Size(min = 1, message = "metricsPath length should be at least 1") + private String metricsPath; + + @JsonProperty(TRACES_PATH) + @Size(min = 1, message = "tracesPath length should be at least 1") + private String tracesPath; + + @JsonProperty(HEALTH_CHECK_SERVICE) + private boolean healthCheck = DEFAULT_HEALTH_CHECK; + + @JsonProperty(PROTO_REFLECTION_SERVICE) + private boolean protoReflectionService = DEFAULT_PROTO_REFLECTION_SERVICE; + + @JsonProperty(ENABLE_UNFRAMED_REQUESTS) + private boolean enableUnframedRequests = DEFAULT_ENABLED_UNFRAMED_REQUESTS; + + @JsonProperty(SSL) + private boolean ssl = DEFAULT_SSL; + + @JsonProperty(OUTPUT_FORMAT) + private OTelOutputFormat outputFormat = OTelOutputFormat.OTEL; + + @JsonProperty(LOGS_OUTPUT_FORMAT) + private OTelOutputFormat logsOutputFormat = null; + + @JsonProperty(METRICS_OUTPUT_FORMAT) + private OTelOutputFormat metricsOutputFormat = null; + + @JsonProperty(TRACES_OUTPUT_FORMAT) + private OTelOutputFormat tracesOutputFormat = null; + + @JsonProperty(USE_ACM_CERT_FOR_SSL) + private boolean useAcmCertForSSL = DEFAULT_USE_ACM_CERT_FOR_SSL; + + @JsonProperty(ACM_CERT_ISSUE_TIME_OUT_MILLIS) + @DurationMin(seconds = 5) + @DurationMax(seconds = 3600) + private Duration acmCertIssueTimeOutMillis = Duration.ofSeconds(DEFAULT_ACM_CERT_ISSUE_TIME_OUT); + + @JsonProperty(SSL_KEY_CERT_FILE) + private String sslKeyCertChainFile; + + @JsonProperty(SSL_KEY_FILE) + private String sslKeyFile; + + private boolean sslCertAndKeyFileInS3; + + @JsonProperty(ACM_CERT_ARN) + private String acmCertificateArn; + + @JsonProperty(ACM_PRIVATE_KEY_PASSWORD) + private String acmPrivateKeyPassword; + + @JsonProperty(AWS_REGION) + private String awsRegion; + + @JsonProperty(THREAD_COUNT) + private int threadCount = DEFAULT_THREAD_COUNT; + + @JsonProperty(MAX_CONNECTION_COUNT) + private int maxConnectionCount = DEFAULT_MAX_CONNECTION_COUNT; + + @JsonProperty("authentication") + private PluginModel authentication; + + @JsonProperty(UNAUTHENTICATED_HEALTH_CHECK) + private boolean unauthenticatedHealthCheck = false; + + @JsonProperty(COMPRESSION) + private CompressionOption compression = CompressionOption.NONE; + + @JsonProperty("max_request_length") + private ByteCount maxRequestLength; + + @JsonProperty(RETRY_INFO) + private RetryInfoConfig retryInfo; + + @AssertTrue(message = LOGS_PATH + " should start with /") + boolean isLogsPathValid() { + return logsPath == null || logsPath.startsWith("/"); + } + + @AssertTrue(message = METRICS_PATH + " should start with /") + boolean isMetricsPathValid() { + return metricsPath == null || metricsPath.startsWith("/"); + } + + @AssertTrue(message = TRACES_PATH + " should start with /") + boolean isTracesPathValid() { + return tracesPath == null || tracesPath.startsWith("/"); + } + + @AssertTrue(message = LOGS_PATH + ", " + METRICS_PATH + ", and " + TRACES_PATH + " should be distinct") + boolean arePathsDistinct() { + if (logsPath == null || metricsPath == null || tracesPath == null) { + return true; // Validation is not applicable if any of the paths are null + } + return !logsPath.equals(metricsPath) && !logsPath.equals(tracesPath) && !metricsPath.equals(tracesPath); + } + + public void validateAndInitializeCertAndKeyFileInS3() { + boolean certAndKeyFileInS3 = false; + if (useAcmCertForSSL) { + validateSSLArgument(String.format("%s is enabled", USE_ACM_CERT_FOR_SSL), acmCertificateArn, ACM_CERT_ARN); + validateSSLArgument(String.format("%s is enabled", USE_ACM_CERT_FOR_SSL), awsRegion, AWS_REGION); + } else if (ssl) { + validateSSLCertificateFiles(); + certAndKeyFileInS3 = isSSLCertificateLocatedInS3(); + if (certAndKeyFileInS3) { + validateSSLArgument("The certificate and key files are located in S3", awsRegion, AWS_REGION); + } + } + sslCertAndKeyFileInS3 = certAndKeyFileInS3; + } + + private void validateSSLArgument(final String sslTypeMessage, final String argument, final String argumentName) { + if (StringUtils.isEmpty(argument)) { + throw new IllegalArgumentException( + String.format("%s, %s can not be empty or null", sslTypeMessage, argumentName)); + } + } + + private void validateSSLCertificateFiles() { + validateSSLArgument(String.format("%s is enabled", SSL), sslKeyCertChainFile, SSL_KEY_CERT_FILE); + validateSSLArgument(String.format("%s is enabled", SSL), sslKeyFile, SSL_KEY_FILE); + } + + private boolean isSSLCertificateLocatedInS3() { + return sslKeyCertChainFile.toLowerCase().startsWith(S3_PREFIX) && + sslKeyFile.toLowerCase().startsWith(S3_PREFIX); + } + + /** + * Note: The value is cast to int since the maximum allowed duration + * (1 hour = 3,600,000 ms) is well within the int range. + * Validation via @DurationMax ensures this is safe. + * Casting to int is necessary because ServerConfiguration method signature + * requires it. + */ + public int getRequestTimeoutInMillis() { + return (int) requestTimeout.toMillis(); + } + + public OTelOutputFormat getLogsOutputFormat() { + return logsOutputFormat != null ? logsOutputFormat : outputFormat; + } + + public OTelOutputFormat getMetricsOutputFormat() { + return metricsOutputFormat != null ? metricsOutputFormat : outputFormat; + } + + public OTelOutputFormat getTracesOutputFormat() { + return tracesOutputFormat != null ? tracesOutputFormat : outputFormat; + } + + public int getPort() { + return port; + } + + public String getLogsPath() { + return logsPath; + } + + public String getMetricsPath() { + return metricsPath; + } + + public String getTracesPath() { + return tracesPath; + } + + public boolean hasHealthCheck() { + return healthCheck; + } + + public boolean enableHttpHealthCheck() { + return enableUnframedRequests() && hasHealthCheck(); + } + + public boolean hasProtoReflectionService() { + return protoReflectionService; + } + + public boolean enableUnframedRequests() { + return enableUnframedRequests; + } + + public boolean isSsl() { + return ssl; + } + + public boolean useAcmCertForSSL() { + return useAcmCertForSSL; + } + + public long getAcmCertIssueTimeOutMillis() { + return acmCertIssueTimeOutMillis.toMillis(); + } + + public String getSslKeyCertChainFile() { + return sslKeyCertChainFile; + } + + public String getSslKeyFile() { + return sslKeyFile; + } + + public String getAcmCertificateArn() { + return acmCertificateArn; + } + + public String getAcmPrivateKeyPassword() { + return acmPrivateKeyPassword; + } + + public boolean isSslCertAndKeyFileInS3() { + return sslCertAndKeyFileInS3; + } + + public String getAwsRegion() { + return awsRegion; + } + + public int getThreadCount() { + return threadCount; + } + + public int getMaxConnectionCount() { + return maxConnectionCount; + } + + public PluginModel getAuthentication() { + return authentication; + } + + public boolean isUnauthenticatedHealthCheck() { + return unauthenticatedHealthCheck; + } + + public CompressionOption getCompression() { + return compression; + } + + public ByteCount getMaxRequestLength() { + return maxRequestLength; + } + + public RetryInfoConfig getRetryInfo() { + return retryInfo; + } + +} diff --git a/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/certificate/CertificateProviderFactory.java b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/certificate/CertificateProviderFactory.java new file mode 100644 index 0000000000..d6ecd67006 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/main/java/org/opensearch/dataprepper/plugins/source/otlp/certificate/CertificateProviderFactory.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otlp.certificate; + +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.file.FileCertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.s3.S3CertificateProvider; +import org.opensearch.dataprepper.plugins.source.otlp.OTLPSourceConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.BackoffStrategy; +import software.amazon.awssdk.core.retry.backoff.EqualJitterBackoffStrategy; +import software.amazon.awssdk.core.retry.conditions.RetryCondition; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.acm.AcmClient; +import software.amazon.awssdk.services.s3.S3Client; + +import java.time.Duration; + +public class CertificateProviderFactory { + private static final Logger LOG = LoggerFactory.getLogger(CertificateProviderFactory.class); + + private static final int ACM_CLIENT_RETRIES = 10; + private static final long ACM_CLIENT_BASE_BACKOFF_MILLIS = 1000L; + private static final long ACM_CLIENT_MAX_BACKOFF_MILLIS = 60000L; + + private final OTLPSourceConfig config; + + public CertificateProviderFactory(final OTLPSourceConfig config) { + this.config = config; + } + + public CertificateProvider getCertificateProvider() { + if (config.useAcmCertForSSL()) { + LOG.info("Using ACM certificate and private key for SSL/TLS."); + final AwsCredentialsProvider credentialsProvider = AwsCredentialsProviderChain.builder() + .addCredentialsProvider(DefaultCredentialsProvider.create()).build(); + + final BackoffStrategy backoffStrategy = EqualJitterBackoffStrategy.builder() + .baseDelay(Duration.ofMillis(ACM_CLIENT_BASE_BACKOFF_MILLIS)) + .maxBackoffTime(Duration.ofMillis(ACM_CLIENT_MAX_BACKOFF_MILLIS)) + .build(); + final RetryPolicy retryPolicy = RetryPolicy.builder() + .numRetries(ACM_CLIENT_RETRIES) + .retryCondition(RetryCondition.defaultRetryCondition()) + .backoffStrategy(backoffStrategy) + .throttlingBackoffStrategy(backoffStrategy) + .build(); + final ClientOverrideConfiguration clientConfig = ClientOverrideConfiguration.builder() + .retryPolicy(retryPolicy) + .build(); + + final AcmClient awsCertificateManager = AcmClient.builder() + .region(Region.of(config.getAwsRegion())) + .credentialsProvider(credentialsProvider) + .overrideConfiguration(clientConfig) + .build(); + + return new ACMCertificateProvider(awsCertificateManager, config.getAcmCertificateArn(), + config.getAcmCertIssueTimeOutMillis(), config.getAcmPrivateKeyPassword()); + } else if (config.isSslCertAndKeyFileInS3()) { + LOG.info("Using S3 to fetch certificate and private key for SSL/TLS."); + final AwsCredentialsProvider credentialsProvider = AwsCredentialsProviderChain.builder() + .addCredentialsProvider(DefaultCredentialsProvider.create()).build(); + final S3Client s3Client = S3Client.builder() + .region(Region.of(config.getAwsRegion())) + .credentialsProvider(credentialsProvider) + .build(); + return new S3CertificateProvider(s3Client, + config.getSslKeyCertChainFile(), + config.getSslKeyFile()); + } else { + LOG.info("Using local file system to get certificate and private key for SSL/TLS."); + return new FileCertificateProvider(config.getSslKeyCertChainFile(), config.getSslKeyFile()); + } + } +} diff --git a/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceConfigTests.java b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceConfigTests.java new file mode 100644 index 0000000000..13c03af689 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceConfigTests.java @@ -0,0 +1,494 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otlp; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.otel.codec.OTelOutputFormat; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class OTLPSourceConfigTests { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); + private static final String PLUGIN_NAME = "otlp"; + private static final String TEST_KEY_CERT = "test.crt"; + private static final String TEST_KEY = "test.key"; + private static final String TEST_KEY_CERT_S3 = "s3://test.crt"; + private static final String TEST_KEY_S3 = "s3://test.key"; + private static final String TEST_REGION = "us-east-1"; + private static final int TEST_PORT = 45600; + private static final int TEST_THREAD_COUNT = 888; + private static final int TEST_MAX_CONNECTION_COUNT = 999; + + private static Stream provideCompressionOption() { + return Stream.of(Arguments.of(CompressionOption.GZIP)); + } + + @Test + void testDefault() { + final OTLPSourceConfig config = new OTLPSourceConfig(); + + assertEquals((int) Duration.ofSeconds(OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT).toMillis(), + config.getRequestTimeoutInMillis()); + assertEquals(OTLPSourceConfig.DEFAULT_PORT, config.getPort()); + assertEquals(OTLPSourceConfig.DEFAULT_THREAD_COUNT, config.getThreadCount()); + assertEquals(OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT, config.getMaxConnectionCount()); + assertEquals(CompressionOption.NONE, config.getCompression()); + assertFalse(config.hasHealthCheck()); + assertFalse(config.hasProtoReflectionService()); + assertFalse(config.enableHttpHealthCheck()); + assertFalse(config.isSslCertAndKeyFileInS3()); + assertTrue(config.isSsl()); + assertNull(config.getSslKeyCertChainFile()); + assertNull(config.getSslKeyFile()); + } + + @ParameterizedTest + @MethodSource("provideCompressionOption") + void testValidCompression(final CompressionOption compressionOption) { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.COMPRESSION, compressionOption.name()); + + final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertEquals(compressionOption, config.getCompression()); + } + + @Test + void testValidConfigWithoutS3CertAndKey() { + final PluginSetting validPluginSetting = completePluginSetting( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, TEST_PORT, + null, true, true, false, true, + TEST_KEY_CERT, TEST_KEY, TEST_THREAD_COUNT, TEST_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(validPluginSetting.getSettings(), + OTLPSourceConfig.class); + config.validateAndInitializeCertAndKeyFileInS3(); + + assertEquals(TEST_PORT, config.getPort()); + assertEquals(TEST_THREAD_COUNT, config.getThreadCount()); + assertEquals(TEST_MAX_CONNECTION_COUNT, config.getMaxConnectionCount()); + assertTrue(config.hasHealthCheck()); + assertTrue(config.hasProtoReflectionService()); + assertFalse(config.enableHttpHealthCheck()); + assertTrue(config.isSsl()); + assertFalse(config.isSslCertAndKeyFileInS3()); + assertEquals(TEST_KEY_CERT, config.getSslKeyCertChainFile()); + assertEquals(TEST_KEY, config.getSslKeyFile()); + } + + @Test + void testValidConfigWithS3CertAndKey() { + final PluginSetting validPluginSettingWithS3CertAndKey = completePluginSetting( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, TEST_PORT, null, false, false, false, + true, TEST_KEY_CERT_S3, TEST_KEY_S3, TEST_THREAD_COUNT, TEST_MAX_CONNECTION_COUNT); + + validPluginSettingWithS3CertAndKey.getSettings().put(OTLPSourceConfig.AWS_REGION, TEST_REGION); + + final OTLPSourceConfig config = OBJECT_MAPPER + .convertValue(validPluginSettingWithS3CertAndKey.getSettings(), OTLPSourceConfig.class); + config.validateAndInitializeCertAndKeyFileInS3(); + + assertEquals(TEST_PORT, config.getPort()); + assertEquals(TEST_THREAD_COUNT, config.getThreadCount()); + assertEquals(TEST_MAX_CONNECTION_COUNT, config.getMaxConnectionCount()); + assertFalse(config.hasHealthCheck()); + assertFalse(config.hasProtoReflectionService()); + assertFalse(config.enableHttpHealthCheck()); + assertTrue(config.isSsl()); + assertTrue(config.isSslCertAndKeyFileInS3()); + assertEquals(TEST_KEY_CERT_S3, config.getSslKeyCertChainFile()); + assertEquals(TEST_KEY_S3, config.getSslKeyFile()); + } + + @Test + void testInvalidConfigWithNullKeyCert() { + final PluginSetting sslNullKeyCertPluginSetting = completePluginSetting( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, OTLPSourceConfig.DEFAULT_PORT, + null, false, false, false, true, null, TEST_KEY, + OTLPSourceConfig.DEFAULT_THREAD_COUNT, OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(sslNullKeyCertPluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertThrows(IllegalArgumentException.class, config::validateAndInitializeCertAndKeyFileInS3); + } + + @Test + void testRetryInfoConfig() { + final PluginSetting pluginSetting = completePluginSetting( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, OTLPSourceConfig.DEFAULT_PORT, + null, false, false, false, true, TEST_KEY_CERT, "", + OTLPSourceConfig.DEFAULT_THREAD_COUNT, OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertThat(config.getRetryInfo().getMaxDelay(), equalTo(Duration.ofMillis(100))); + assertThat(config.getRetryInfo().getMinDelay(), equalTo(Duration.ofMillis(50))); + } + + @Test + void testHttpHealthCheckWithUnframedRequestEnabled() { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.ENABLE_UNFRAMED_REQUESTS, "true"); + settings.put(OTLPSourceConfig.HEALTH_CHECK_SERVICE, "true"); + settings.put(OTLPSourceConfig.PROTO_REFLECTION_SERVICE, "true"); + + final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertTrue(config.hasHealthCheck()); + assertTrue(config.enableUnframedRequests()); + assertTrue(config.hasProtoReflectionService()); + assertTrue(config.enableHttpHealthCheck()); + } + + @Test + void testHttpHealthCheckWithUnframedRequestDisabled() { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.ENABLE_UNFRAMED_REQUESTS, "false"); + settings.put(OTLPSourceConfig.HEALTH_CHECK_SERVICE, "true"); + settings.put(OTLPSourceConfig.PROTO_REFLECTION_SERVICE, "true"); + + final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertTrue(config.hasHealthCheck()); + assertFalse(config.enableUnframedRequests()); + assertTrue(config.hasProtoReflectionService()); + assertFalse(config.enableHttpHealthCheck()); + } + + @Test + void testInvalidConfigWithEmptyKeyCert() { + final PluginSetting sslEmptyKeyCertPluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + null, + false, + false, + false, + true, + "", + TEST_KEY, + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(sslEmptyKeyCertPluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertThrows(IllegalArgumentException.class, config::validateAndInitializeCertAndKeyFileInS3); + } + + @Test + void testInvalidConfigWithEmptyKeyFile() { + final PluginSetting sslEmptyKeyFilePluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + null, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(sslEmptyKeyFilePluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertThrows(IllegalArgumentException.class, config::validateAndInitializeCertAndKeyFileInS3); + } + + @Test + void testValidConfigWithCustomPath() { + final String testPath = "/testPath"; + final PluginSetting customPathPluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + testPath, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertThat(config.getLogsPath(), equalTo(testPath)); + assertThat(config.isLogsPathValid(), equalTo(true)); + } + + @Test + void testInValidConfigWithCustomPath() { + final String testPath = "invalidPath"; + final PluginSetting customPathPluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + testPath, + false, + false, + false, + true, + TEST_KEY_CERT, + "", + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertThat(config.getLogsPath(), equalTo(testPath)); + assertThat(config.isLogsPathValid(), equalTo(false)); + } + + @Test + void testPathsAreDistinct() { + final PluginSetting pluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + "/logs", + false, + false, + false, + true, + TEST_KEY_CERT, + TEST_KEY, + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + pluginSetting.getSettings().put(OTLPSourceConfig.METRICS_PATH, "/metrics"); + pluginSetting.getSettings().put(OTLPSourceConfig.TRACES_PATH, "/traces"); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertTrue(config.arePathsDistinct()); + } + + @Test + void testPathsAreNotDistinct() { + final PluginSetting pluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + "/logs", + false, + false, + false, + true, + TEST_KEY_CERT, + TEST_KEY, + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + pluginSetting.getSettings().put(OTLPSourceConfig.METRICS_PATH, "/logs"); + pluginSetting.getSettings().put(OTLPSourceConfig.TRACES_PATH, "/traces"); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertFalse(config.arePathsDistinct()); + } + + @Test + void testValidPathsStartWithSlash() { + final PluginSetting pluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + "/logs", + false, + false, + false, + true, + TEST_KEY_CERT, + TEST_KEY, + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + pluginSetting.getSettings().put(OTLPSourceConfig.METRICS_PATH, "/metrics"); + pluginSetting.getSettings().put(OTLPSourceConfig.TRACES_PATH, "/traces"); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertTrue(config.isLogsPathValid()); + assertTrue(config.isMetricsPathValid()); + assertTrue(config.isTracesPathValid()); + } + + @Test + void testInvalidPathsDoNotStartWithSlash() { + final PluginSetting pluginSetting = completePluginSettingForOtelTelemetrySource( + OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT, + OTLPSourceConfig.DEFAULT_PORT, + "logs", + false, + false, + false, + true, + TEST_KEY_CERT, + TEST_KEY, + OTLPSourceConfig.DEFAULT_THREAD_COUNT, + OTLPSourceConfig.DEFAULT_MAX_CONNECTION_COUNT); + pluginSetting.getSettings().put(OTLPSourceConfig.METRICS_PATH, "metrics"); + pluginSetting.getSettings().put(OTLPSourceConfig.TRACES_PATH, "traces"); + + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + assertFalse(config.isLogsPathValid()); + assertFalse(config.isMetricsPathValid()); + assertFalse(config.isTracesPathValid()); + } + + @Test + void testDefaultOutputFormats() { + final OTLPSourceConfig config = new OTLPSourceConfig(); + + assertEquals(OTelOutputFormat.OTEL, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OTEL, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OTEL, config.getTracesOutputFormat()); + } + + @Test + void testGenericOutputFormats() { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.OUTPUT_FORMAT, OTelOutputFormat.OPENSEARCH.getFormatName()); + + final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTLPSourceConfig.class); + + assertEquals(OTelOutputFormat.OPENSEARCH, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getTracesOutputFormat()); + } + + @Test + void testCustomOutputFormats() { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.LOGS_OUTPUT_FORMAT, OTelOutputFormat.OPENSEARCH.getFormatName()); + settings.put(OTLPSourceConfig.METRICS_OUTPUT_FORMAT, OTelOutputFormat.OPENSEARCH.getFormatName()); + settings.put(OTLPSourceConfig.TRACES_OUTPUT_FORMAT, OTelOutputFormat.OPENSEARCH.getFormatName()); + + final PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + final OTLPSourceConfig config = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTLPSourceConfig.class); + + assertEquals(OTelOutputFormat.OPENSEARCH, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getTracesOutputFormat()); + } + + @Test + void testOutputFormatFallbackAndOverride() { + // Default: all specific formats are null, should fallback to outputFormat + // (OTEL) + OTLPSourceConfig config = new OTLPSourceConfig(); + assertEquals(OTelOutputFormat.OTEL, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OTEL, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OTEL, config.getTracesOutputFormat()); + + // Set generic outputFormat to OPENSEARCH, specific formats still null + Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.OUTPUT_FORMAT, OTelOutputFormat.OPENSEARCH.getFormatName()); + PluginSetting pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + config = new ObjectMapper().convertValue(pluginSetting.getSettings(), OTLPSourceConfig.class); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getTracesOutputFormat()); + + // Set specific logs output format, others remain null + settings.put(OTLPSourceConfig.LOGS_OUTPUT_FORMAT, OTelOutputFormat.OTEL.getFormatName()); + pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + config = new ObjectMapper().convertValue(pluginSetting.getSettings(), OTLPSourceConfig.class); + assertEquals(OTelOutputFormat.OTEL, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getTracesOutputFormat()); + + // Set all specific formats + settings.put(OTLPSourceConfig.METRICS_OUTPUT_FORMAT, OTelOutputFormat.OTEL.getFormatName()); + settings.put(OTLPSourceConfig.TRACES_OUTPUT_FORMAT, OTelOutputFormat.OPENSEARCH.getFormatName()); + pluginSetting = new PluginSetting(PLUGIN_NAME, settings); + config = new ObjectMapper().convertValue(pluginSetting.getSettings(), OTLPSourceConfig.class); + assertEquals(OTelOutputFormat.OTEL, config.getLogsOutputFormat()); + assertEquals(OTelOutputFormat.OTEL, config.getMetricsOutputFormat()); + assertEquals(OTelOutputFormat.OPENSEARCH, config.getTracesOutputFormat()); + } + + private PluginSetting completePluginSetting(final int requestTimeoutInMillis, + final int port, final String path, + final boolean healthCheck, final boolean protoReflectionService, + final boolean enableUnframedRequests, final boolean isSSL, + final String sslKeyCertChainFile, final String sslKeyFile, + final int threadCount, final int maxConnectionCount) { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.PORT, port); + settings.put(OTLPSourceConfig.LOGS_PATH, path); + settings.put(OTLPSourceConfig.HEALTH_CHECK_SERVICE, healthCheck); + settings.put(OTLPSourceConfig.PROTO_REFLECTION_SERVICE, protoReflectionService); + settings.put(OTLPSourceConfig.ENABLE_UNFRAMED_REQUESTS, enableUnframedRequests); + settings.put(OTLPSourceConfig.SSL, isSSL); + settings.put(OTLPSourceConfig.SSL_KEY_CERT_FILE, sslKeyCertChainFile); + settings.put(OTLPSourceConfig.SSL_KEY_FILE, sslKeyFile); + settings.put(OTLPSourceConfig.THREAD_COUNT, threadCount); + settings.put(OTLPSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); + settings.put(OTLPSourceConfig.RETRY_INFO, + new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); + return new PluginSetting(PLUGIN_NAME, settings); + } + + private PluginSetting completePluginSettingForOtelTelemetrySource( + final int requestTimeoutInMillis, + final int port, + final String path, + final boolean healthCheck, + final boolean protoReflectionService, + final boolean enableUnframedRequests, + final boolean isSSL, + final String sslKeyCertChainFile, + final String sslKeyFile, + final int threadCount, + final int maxConnectionCount) { + final Map settings = new HashMap<>(); + settings.put(OTLPSourceConfig.PORT, port); + settings.put(OTLPSourceConfig.LOGS_PATH, path); + settings.put(OTLPSourceConfig.HEALTH_CHECK_SERVICE, healthCheck); + settings.put(OTLPSourceConfig.PROTO_REFLECTION_SERVICE, protoReflectionService); + settings.put(OTLPSourceConfig.ENABLE_UNFRAMED_REQUESTS, enableUnframedRequests); + settings.put(OTLPSourceConfig.SSL, isSSL); + settings.put(OTLPSourceConfig.SSL_KEY_CERT_FILE, sslKeyCertChainFile); + settings.put(OTLPSourceConfig.SSL_KEY_FILE, sslKeyFile); + settings.put(OTLPSourceConfig.THREAD_COUNT, threadCount); + settings.put(OTLPSourceConfig.MAX_CONNECTION_COUNT, maxConnectionCount); + settings.put(OTLPSourceConfig.RETRY_INFO, + new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(100))); + return new PluginSetting(PLUGIN_NAME, settings); + } +} diff --git a/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceTest.java b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceTest.java new file mode 100644 index 0000000000..252dbc335f --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSourceTest.java @@ -0,0 +1,1821 @@ +package org.opensearch.dataprepper.plugins.source.otlp; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.StringJoiner; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; +import org.opensearch.dataprepper.metrics.MetricNames; +import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.types.ByteCount; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.server.HealthGrpcService; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; +import org.opensearch.dataprepper.plugins.source.otlp.certificate.CertificateProviderFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.ClosedSessionException; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.server.HttpService; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; + +import io.grpc.BindableService; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.instrument.Measurement; +import io.micrometer.core.instrument.Statistic; +import io.netty.util.AsciiString; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceResponse; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.InstrumentationScope; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; +import io.opentelemetry.proto.metrics.v1.Gauge; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; +import io.opentelemetry.proto.resource.v1.Resource; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.otlp.OTLPSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.otlp.OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT; +import static org.opensearch.dataprepper.plugins.source.otlp.OTLPSourceConfig.SSL; + +@ExtendWith(MockitoExtension.class) +class OTLPSourceTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21893/"; + private static final String USERNAME = "test_user"; + private static final String PASSWORD = "test_password"; + private static final String LOGS_TEST_PATH = "${pipelineName}/v1/logs"; + private static final String METRICS_TEST_PATH = "${pipelineName}/v1/metrics"; + private static final String TRACES_TEST_PATH = "${pipelineName}/v1/traces"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), + Duration.ofMillis(2000)); + + private static final ExportLogsServiceRequest LOGS_REQUEST = ExportLogsServiceRequest.newBuilder() + .addResourceLogs(ResourceLogs.newBuilder().build()).build(); + private static final ExportMetricsServiceRequest METRICS_REQUEST = ExportMetricsServiceRequest.newBuilder() + .addResourceMetrics(ResourceMetrics.newBuilder().build()).build(); + private static final ExportTraceServiceRequest TRACES_SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() + .addSpans( + io.opentelemetry.proto.trace.v1.Span + .newBuilder() + .setTraceState("SUCCESS") + .build())) + .build()) + .build(); + private static final ExportTraceServiceRequest TRACES_FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() + .addSpans( + io.opentelemetry.proto.trace.v1.Span + .newBuilder() + .setTraceState("FAILURE") + .build())) + .build()) + .build(); + + @Mock + private ServerBuilder serverBuilder; + + @Mock + private Server server; + + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + + @Mock + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; + + @Mock + private Certificate certificate; + + @Mock + private CompletableFuture completableFuture; + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTLPSourceConfig otlpSourceConfig; + + @Mock + private BlockingBuffer> buffer; + + @Mock + private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; + + private PluginSetting pluginSetting; + private PluginSetting testPluginSetting; + private PluginMetrics pluginMetrics; + private PipelineDescription pipelineDescription; + private OTLPSource SOURCE; + + @BeforeEach + public void beforeEach() { + lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.service(any(GrpcService.class), any(Function.class))) + .thenReturn(serverBuilder); + lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.build()).thenReturn(server); + lenient().when(server.start()).thenReturn(completableFuture); + + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))) + .thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + + when(otlpSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(otlpSourceConfig.isSsl()).thenReturn(false); + when(otlpSourceConfig.getRequestTimeoutInMillis()) + .thenReturn((int) Duration.ofSeconds(DEFAULT_REQUEST_TIMEOUT).toMillis()); + when(otlpSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(otlpSourceConfig.getThreadCount()).thenReturn(5); + when(otlpSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(otlpSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + configureObjectUnderTest(); + pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + } + + @AfterEach + public void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + MetricsTestUtil.initMetrics(); + pluginMetrics = PluginMetrics.fromNames("otlp-source", "pipeline"); + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTLPSource(otlpSourceConfig, pluginMetrics, pluginFactory, + pipelineDescription); + } + + @Test + void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(TRACES_SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(TRACES_FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(LOGS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpsFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", false); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + pluginSetting = new PluginSetting("otlp-source", settingsMap); + pluginSetting.setPipelineName("pipeline"); + + otlpSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + SOURCE = new OTLPSource(otlpSourceConfig, pluginMetrics, pluginFactory, + pipelineDescription); + + SOURCE.start(buffer); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(TRACES_SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(TRACES_FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(LOGS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullBytesWithNonUnframedRequests() { + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(TRACES_SUCCESS_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(TRACES_FAILURE_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(METRICS_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(LOGS_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithUnframedRequests() throws InvalidProtocolBufferException { + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(LOGS_REQUEST).getBytes())) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpCompressionWithUnframedRequests() throws IOException { + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getCompression()).thenReturn(CompressionOption.GZIP); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") + .build(), + createGZipCompressedPayload(JsonFormat.printer().print(createExportTraceRequest()))) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") + .build(), + createGZipCompressedPayload(JsonFormat.printer().print(METRICS_REQUEST))) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.JSON_UTF_8) + .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") + .build(), + createGZipCompressedPayload(JsonFormat.printer().print(LOGS_REQUEST))) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolBufferException { + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getLogsPath()).thenReturn(LOGS_TEST_PATH); + when(otlpSourceConfig.getMetricsPath()).thenReturn(METRICS_TEST_PATH); + when(otlpSourceConfig.getTracesPath()).thenReturn(TRACES_TEST_PATH); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String transformedTracesPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path(transformedTracesPath) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + final String transformedMetricsPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path(transformedMetricsPath) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportMetricsRequest()).getBytes())) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + final String transformedLogsPath = "/" + TEST_PIPELINE_NAME + "/v1/logs"; + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path(transformedLogsPath) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportLogsRequest()).getBytes())) + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndAuthHeader_with_successful_response() + throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider( + httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(otlpSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD))); + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getLogsPath()).thenReturn(LOGS_TEST_PATH); + when(otlpSourceConfig.getMetricsPath()).thenReturn(METRICS_TEST_PATH); + when(otlpSourceConfig.getTracesPath()).thenReturn(TRACES_TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String encodeToString = Base64.getEncoder() + .encodeToString(String.format("%s:%s", USERNAME, PASSWORD) + .getBytes(StandardCharsets.UTF_8)); + + final String transformedTracesPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedTracesPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportTraceRequest()).getBytes()) + .header("Authorization", "Basic " + encodeToString) + .execute() + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + final String transformedMetricsPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedMetricsPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) + .header("Authorization", "Basic " + encodeToString) + .execute() + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + final String transformedLogsPath = "/" + TEST_PIPELINE_NAME + "/v1/logs"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedLogsPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportLogsRequest()).getBytes()) + .header("Authorization", "Basic " + encodeToString) + .execute() + .aggregate() + .whenComplete( + (response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndAuthHeader_with_unsuccessful_response() + throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider( + httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(otlpSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD))); + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getLogsPath()).thenReturn(LOGS_TEST_PATH); + when(otlpSourceConfig.getMetricsPath()).thenReturn(METRICS_TEST_PATH); + when(otlpSourceConfig.getTracesPath()).thenReturn(TRACES_TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String transformedTracesPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedTracesPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportTraceRequest()).getBytes()) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNAUTHORIZED, throwable)) + .join(); + + final String transformedMetricsPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedMetricsPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNAUTHORIZED, throwable)) + .join(); + + final String transformedLogsPath = "/" + TEST_PIPELINE_NAME + "/v1/logs"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedLogsPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportLogsRequest()).getBytes()) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNAUTHORIZED, throwable)) + .join(); + } + + @Test + void testHttpRequestWithInvalidCredentialsShouldReturnUnauthorized() throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider( + httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(otlpSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD))); + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getLogsPath()).thenReturn(LOGS_TEST_PATH); + when(otlpSourceConfig.getMetricsPath()).thenReturn(METRICS_TEST_PATH); + when(otlpSourceConfig.getTracesPath()).thenReturn(TRACES_TEST_PATH); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String invalidUsername = "wrong_user"; + final String invalidPassword = "wrong_password"; + final String invalidCredentials = Base64.getEncoder() + .encodeToString( + String.format("%s:%s", invalidUsername, invalidPassword) + .getBytes(StandardCharsets.UTF_8)); + + final String transformedTracesPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedTracesPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportTraceRequest()).getBytes()) + .header("Authorization", "Basic " + invalidCredentials) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNAUTHORIZED, throwable)) + .join(); + + final String transformedMetricsPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedMetricsPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) + .header("Authorization", "Basic " + invalidCredentials) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNAUTHORIZED, throwable)) + .join(); + + final String transformedLogsPath = "/" + TEST_PIPELINE_NAME + "/v1/logs"; + WebClient.of().prepare() + .post("http://127.0.0.1:21893" + transformedLogsPath) + .content(MediaType.JSON_UTF_8, + JsonFormat.printer().print(createExportLogsRequest()).getBytes()) + .header("Authorization", "Basic " + invalidCredentials) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.UNAUTHORIZED, throwable)) + .join(); + } + + @Test + void testGrpcRequestWithoutAuthentication_with_unsuccessful_response() { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider( + httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(otlpSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD))); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub tracesClient = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final StatusRuntimeException tracesActualException = assertThrows(StatusRuntimeException.class, + () -> tracesClient.export(createExportTraceRequest())); + assertThat(tracesActualException.getStatus(), notNullValue()); + assertThat(tracesActualException.getStatus().getCode(), equalTo(Status.Code.UNAUTHENTICATED)); + + final MetricsServiceGrpc.MetricsServiceBlockingStub metricsClient = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final StatusRuntimeException metricsActualException = assertThrows(StatusRuntimeException.class, + () -> metricsClient.export(createExportMetricsRequest())); + assertThat(metricsActualException.getStatus(), notNullValue()); + assertThat(metricsActualException.getStatus().getCode(), equalTo(Status.Code.UNAUTHENTICATED)); + + final LogsServiceGrpc.LogsServiceBlockingStub logsClient = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + final StatusRuntimeException logsActualException = assertThrows(StatusRuntimeException.class, + () -> logsClient.export(createExportLogsRequest())); + assertThat(logsActualException.getStatus(), notNullValue()); + assertThat(logsActualException.getStatus().getCode(), equalTo(Status.Code.UNAUTHENTICATED)); + } + + @Test + void testHttpWithoutSslFailsWhenSslIsEnabled() throws InvalidProtocolBufferException { + when(otlpSourceConfig.isSsl()).thenReturn(true); + when(otlpSourceConfig.getSslKeyCertChainFile()) + .thenReturn("src/test/resources/certificate/test_cert.crt"); + when(otlpSourceConfig.getSslKeyFile()) + .thenReturn("src/test/resources/certificate/test_decrypted_key.key"); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient client = WebClient.builder("http://127.0.0.1:21893") + .build(); + + CompletionException tracesException = assertThrows(CompletionException.class, + () -> client.execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()) + .getBytes())) + .aggregate() + .join()); + assertThat(tracesException.getCause(), instanceOf(ClosedSessionException.class)); + + CompletionException metricsException = assertThrows(CompletionException.class, + () -> client.execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportMetricsRequest()) + .getBytes())) + .aggregate() + .join()); + assertThat(metricsException.getCause(), instanceOf(ClosedSessionException.class)); + + CompletionException logsException = assertThrows(CompletionException.class, + () -> client.execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportLogsRequest()) + .getBytes())) + .aggregate() + .join()); + assertThat(logsException.getCause(), instanceOf(ClosedSessionException.class)); + } + + @Test + void testGrpcFailsIfSslIsEnabledAndNoTls() { + when(otlpSourceConfig.isSsl()).thenReturn(true); + when(otlpSourceConfig.getSslKeyCertChainFile()) + .thenReturn("src/test/resources/certificate/test_cert.crt"); + when(otlpSourceConfig.getSslKeyFile()) + .thenReturn("src/test/resources/certificate/test_decrypted_key.key"); + configureObjectUnderTest(); + SOURCE.start(buffer); + + TraceServiceGrpc.TraceServiceBlockingStub tracesClient = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + + StatusRuntimeException tracesActualException = assertThrows(StatusRuntimeException.class, + () -> tracesClient.export(createExportTraceRequest())); + + assertThat(tracesActualException.getStatus(), notNullValue()); + assertThat(tracesActualException.getStatus().getCode(), equalTo(Status.Code.UNKNOWN)); + + final MetricsServiceGrpc.MetricsServiceBlockingStub metricsClient = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final StatusRuntimeException metricsActualException = assertThrows(StatusRuntimeException.class, + () -> metricsClient.export(createExportMetricsRequest())); + assertThat(metricsActualException.getStatus(), notNullValue()); + assertThat(metricsActualException.getStatus().getCode(), equalTo(Status.Code.UNKNOWN)); + + final LogsServiceGrpc.LogsServiceBlockingStub logsClient = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + + final StatusRuntimeException logsActualException = assertThrows(StatusRuntimeException.class, + () -> logsClient.export(createExportLogsRequest())); + + assertThat(logsActualException.getStatus(), notNullValue()); + assertThat(logsActualException.getStatus().getCode(), equalTo(Status.Code.UNKNOWN)); + + } + + @Test + void testServerStartCertFileSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + + final Path certFilePath = Path.of("src/test/resources/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("src/test/resources/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", false); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, + pluginMetrics, pluginFactory, pipelineDescription); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), + StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), + StandardCharsets.UTF_8.name()); + assertThat(actualCertificate, is(certAsString)); + assertThat(actualPrivateKey, is(keyAsString)); + } + } + + @Test + void testServerStartACMCertSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("src/test/resources/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("src/test/resources/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("acm_certificate_arn", + "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, + pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), + StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), + StandardCharsets.UTF_8.name()); + assertThat(actualCertificate, is(certAsString)); + assertThat(actualPrivateKey, is(keyAsString)); + } + } + + @Test + void start_with_Health_configured_includes_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))) + .thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("src/test/resources/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("src/test/resources/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("acm_certificate_arn", + "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, + pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"), isA(HealthCheckService.class)); + } + + @Test + void start_with_Health_configured_unframed_requests_includes_HTTPHealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))) + .thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("src/test/resources/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("src/test/resources/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("acm_certificate_arn", + "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, + pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder).service(eq("/health"), isA(HealthCheckService.class)); + } + + @Test + void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))) + .thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("src/test/resources/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("src/test/resources/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("acm_certificate_arn", + "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, + pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"), isA(HealthCheckService.class)); + } + + @Test + void start_without_Health_configured_unframed_requests_does_not_include_HealthCheck_service() + throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))) + .thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("src/test/resources/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("src/test/resources/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("use_acm_certificate_for_ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("acm_certificate_arn", + "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, + pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"), isA(HealthCheckService.class)); + } + + @Test + void testHealthCheckUnauthNotAllowed() throws InterruptedException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", false); + settingsMap.put("port", 21893); // Explicitly set the port + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + settingsMap.put("proto_reflection_service", "true"); + settingsMap.put("unauthenticated_health_check", "false"); + settingsMap.put("authentication", new PluginModel("http_basic", + Map.of( + "username", "test", + "password", "test2"))); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, certificateProviderFactory, pipelineDescription); + + source.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("localhost:21893") + .method(HttpMethod.GET) + .path("/health") + .build()) + .aggregate() + .whenComplete((response, throwable) -> { + assertNotNull(response, "Response should not be null"); + assertEquals(HttpStatus.UNAUTHORIZED, response.status(), + "Expected UNAUTHORIZED status"); + assertNull(throwable, "Throwable should be null"); + }) + .join(); + + source.stop(); + } + + @Test + void testHealthCheckUnauthAllowed() { + // Prepare + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, false); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + settingsMap.put("proto_reflection_service", "true"); + settingsMap.put("unauthenticated_health_check", "true"); + settingsMap.put("authentication", new PluginModel("http_basic", + Map.of( + "username", "test", + "password", "test2"))); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, certificateProviderFactory, pipelineDescription); + + source.start(buffer); + + // When + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("localhost:21893") + .method(HttpMethod.GET) + .path("/health") + .build()) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + source.stop(); + } + + @Test + void testOptionalHttpAuthServiceNotInPlace() { + when(server.stop()).thenReturn(completableFuture); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + SOURCE.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder, never()).decorator(isA(Function.class)); + } + + @Test + void testOptionalHttpAuthServiceInPlace() { + final Optional> function = Optional + .of(httpService -> httpService); + + final Map settingsMap = new HashMap<>(); + settingsMap.put("authentication", new PluginModel("test", null)); + settingsMap.put("unauthenticated_health_check", true); + + settingsMap.put(SSL, false); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + + when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); + + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, certificateProviderFactory, pipelineDescription); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder).decorator(isA(String.class), isA(Function.class)); + } + + @Test + void testOptionalHttpAuthServiceInPlaceWithUnauthenticatedDisabled() { + final Optional> function = Optional + .of(httpService -> httpService); + + final Map settingsMap = new HashMap<>(); + settingsMap.put("authentication", new PluginModel("test", null)); + settingsMap.put("unauthenticated_health_check", false); + + settingsMap.put(SSL, false); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + + when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); + + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, certificateProviderFactory, pipelineDescription); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder).decorator(isA(Function.class)); + } + + @Test + void testDoubleStart() { + // starting server + SOURCE.start(buffer); + // double start server + assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); + } + + @Test + void testRunAnotherSourceWithSamePort() { + // starting server + SOURCE.start(buffer); + + Map settingsMap = Map.of("retry_info", TEST_RETRY_INFO, SSL, false); + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + // Expect RuntimeException because when port is already in use, BindException is + // thrown which is not RuntimeException + assertThrows(RuntimeException.class, () -> source.start(buffer)); + } + + @Test + void testStartWithEmptyBuffer() { + testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); + testPluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), + OTLPSourceConfig.class); + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + assertThrows(IllegalStateException.class, () -> source.start(null)); + } + + @Test + void testStartWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { + // Prepare + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + final NullPointerException expCause = new NullPointerException(); + when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); + + // When/Then + final RuntimeException ex = assertThrows(RuntimeException.class, () -> source.start(buffer)); + assertEquals(expCause, ex); + } + } + + @Test + void testStopWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { + // Prepare + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + + // When/Then + when(completableFuture.get()).thenThrow(new ExecutionException("", null)); + assertThrows(RuntimeException.class, source::stop); + } + } + + @Test + void testStartWithInterruptedException() throws ExecutionException, InterruptedException { + // Prepare + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(completableFuture.get()).thenThrow(new InterruptedException()); + + // When/Then + assertThrows(RuntimeException.class, () -> source.start(buffer)); + assertTrue(Thread.interrupted()); + } + } + + @Test + void testStopWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { + // Prepare + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + final NullPointerException expCause = new NullPointerException(); + when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); + + // When/Then + final RuntimeException ex = assertThrows(RuntimeException.class, source::stop); + assertEquals(expCause, ex); + } + } + + @Test + void testStopWithInterruptedException() throws ExecutionException, InterruptedException { + // Prepare + final OTLPSource source = new OTLPSource(otlpSourceConfig, pluginMetrics, + pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + when(completableFuture.get()).thenThrow(new InterruptedException()); + + // When/Then + assertThrows(RuntimeException.class, source::stop); + assertTrue(Thread.interrupted()); + } + } + + @Test + void gRPC_request_writes_to_buffer_with_successful_response() throws Exception { + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); + assertThat(exportResponse, notNullValue()); + + final MetricsServiceGrpc.MetricsServiceBlockingStub metricsClient = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final ExportMetricsServiceResponse metricsResponse = metricsClient.export(createExportMetricsRequest()); + assertThat(metricsResponse, notNullValue()); + + final LogsServiceGrpc.LogsServiceBlockingStub logsClient = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + final ExportLogsServiceResponse logsResponse = logsClient.export(createExportLogsRequest()); + assertThat(logsResponse, notNullValue()); + + final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor + .forClass(Collection.class); + verify(buffer, times(3)).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); + + final List>> allBufferWrites = bufferWriteArgumentCaptor.getAllValues(); + assertThat(allBufferWrites, hasSize(3)); + } + + @Test + void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws Exception { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider( + httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD))); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String encodeToString = Base64.getEncoder() + .encodeToString(String.format("%s:%s", USERNAME, PASSWORD) + .getBytes(StandardCharsets.UTF_8)); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .addHeader("Authorization", "Basic " + encodeToString) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); + assertThat(exportResponse, notNullValue()); + + final MetricsServiceGrpc.MetricsServiceBlockingStub metricsClient = Clients.builder(GRPC_ENDPOINT) + .addHeader("Authorization", "Basic " + encodeToString) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final ExportMetricsServiceResponse metricsResponse = metricsClient.export(createExportMetricsRequest()); + assertThat(metricsResponse, notNullValue()); + + final LogsServiceGrpc.LogsServiceBlockingStub logsClient = Clients.builder(GRPC_ENDPOINT) + .addHeader("Authorization", "Basic " + encodeToString) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + final ExportLogsServiceResponse logsResponse = logsClient.export(createExportLogsRequest()); + assertThat(logsResponse, notNullValue()); + + final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor + .forClass(Collection.class); + verify(buffer, times(3)).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); + + final List>> allBufferWrites = bufferWriteArgumentCaptor.getAllValues(); + assertThat(allBufferWrites, hasSize(3)); + } + + @Test + void gRPC_request_with_custom_path_throws_when_written_to_default_path() { + when(otlpSourceConfig.getTracesPath()).thenReturn(TRACES_TEST_PATH); + when(otlpSourceConfig.getMetricsPath()).thenReturn(METRICS_TEST_PATH); + when(otlpSourceConfig.getLogsPath()).thenReturn(LOGS_TEST_PATH); + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, + () -> client.export(createExportTraceRequest())); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); + } + + @ParameterizedTest + @ArgumentsSource(BufferExceptionToStatusArgumentsProvider.class) + void grpc_request_returns_expected_status_for_exceptions_from_buffer( + final Class bufferExceptionClass, + final Status.Code expectedStatusCode) throws Exception { + + configureObjectUnderTest(); + SOURCE.start(buffer); + + // Simulate the buffer throwing the provided exception + doThrow(bufferExceptionClass).when(buffer).writeAll(anyCollection(), anyInt()); + + final TraceServiceGrpc.TraceServiceBlockingStub traceClient = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final ExportTraceServiceRequest traceRequest = createExportTraceRequest(); + final StatusRuntimeException traceException = assertThrows(StatusRuntimeException.class, + () -> traceClient.export(traceRequest)); + assertThat(traceException.getStatus().getCode(), equalTo(expectedStatusCode)); + + final MetricsServiceGrpc.MetricsServiceBlockingStub metricsClient = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final ExportMetricsServiceRequest metricsRequest = createExportMetricsRequest(); + final StatusRuntimeException metricsException = assertThrows(StatusRuntimeException.class, + () -> metricsClient.export(metricsRequest)); + assertThat(metricsException.getStatus().getCode(), equalTo(expectedStatusCode)); + + final LogsServiceGrpc.LogsServiceBlockingStub logsClient = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + final ExportLogsServiceRequest logsRequest = createExportLogsRequest(); + final StatusRuntimeException logsException = assertThrows(StatusRuntimeException.class, + () -> logsClient.export(logsRequest)); + assertThat(logsException.getStatus().getCode(), equalTo(expectedStatusCode)); + } + + @Test + void gRPC_request_throws_InvalidArgument_for_malformed_data() { + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final ExportTraceServiceRequest exportTraceRequest = createInvalidExportTraceRequest(); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, + () -> client.export(exportTraceRequest)); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.INVALID_ARGUMENT)); + verifyNoInteractions(buffer); + } + + @Test + void request_that_exceeds_maxRequestLength_returns_413() throws InvalidProtocolBufferException { + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + when(otlpSourceConfig.getMaxRequestLength()).thenReturn(ByteCount.ofBytes(4)); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.logs.v1.LogsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportLogsRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) + .join(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportMetricsRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) + .join(); + } + + @Test + void testServerConnectionsMetric() throws InvalidProtocolBufferException { + // Prepare + when(otlpSourceConfig.enableUnframedRequests()).thenReturn(true); + SOURCE.start(buffer); + + final String metricNamePrefix = new StringJoiner(MetricNames.DELIMITER) + .add("pipeline").add("otlp-source").toString(); + List serverConnectionsMeasurements = MetricsTestUtil.getMeasurementList( + new StringJoiner(MetricNames.DELIMITER).add(metricNamePrefix) + .add(OTLPSource.SERVER_CONNECTIONS).toString()); + + // Verify connections metric value is 0 + Measurement serverConnectionsMeasurement = MetricsTestUtil + .getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); + assertEquals(0, serverConnectionsMeasurement.getValue()); + + final RequestHeaders testRequestHeaders = RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21893") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(); + final HttpData testHttpData = HttpData + .copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes()); + + // Send request + WebClient.of().execute(testRequestHeaders, testHttpData) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, + HttpStatus.OK, throwable)) + .join(); + + // Verify connections metric value is 1 + serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, + Statistic.VALUE); + assertEquals(1.0, serverConnectionsMeasurement.getValue()); + } + + static class BufferExceptionToStatusArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + arguments(TimeoutException.class, Status.Code.RESOURCE_EXHAUSTED), + arguments(SizeOverflowException.class, Status.Code.RESOURCE_EXHAUSTED), + arguments(Exception.class, Status.Code.INTERNAL), + arguments(RuntimeException.class, Status.Code.INTERNAL)); + } + } + + private ExportLogsServiceRequest createExportLogsRequest() { + final Resource resource = Resource.newBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey("service.name") + .setValue(AnyValue.newBuilder().setStringValue("service").build())) + .build(); + + final ResourceLogs resourceLogs = ResourceLogs.newBuilder() + .addScopeLogs(ScopeLogs.newBuilder() + .addLogRecords(LogRecord.newBuilder().setSeverityNumberValue(1)) + .build()) + .setResource(resource) + .build(); + + return ExportLogsServiceRequest.newBuilder() + .addResourceLogs(resourceLogs) + .build(); + } + + private ExportMetricsServiceRequest createExportMetricsRequest() { + final Resource resource = Resource.newBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey("service.name") + .setValue(AnyValue.newBuilder().setStringValue("service").build())) + .build(); + NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); + Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); + + io.opentelemetry.proto.metrics.v1.Metric.Builder metric = io.opentelemetry.proto.metrics.v1.Metric + .newBuilder() + .setGauge(gauge) + .setUnit("seconds") + .setName("name") + .setDescription("description"); + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() + .addMetrics(metric) + .setScope(InstrumentationScope.newBuilder() + .setName("ilname") + .setVersion("ilversion") + .build()) + .build(); + + final ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() + .setResource(resource) + .addScopeMetrics(scopeMetrics) + .build(); + + return ExportMetricsServiceRequest.newBuilder() + .addResourceMetrics(resourceMetrics).build(); + } + + private ExportTraceServiceRequest createInvalidExportTraceRequest() { + final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() + .setTraceState("SUCCESS").build(); + final ExportTraceServiceRequest successRequest = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + + return successRequest; + } + + private ExportTraceServiceRequest createExportTraceRequest() { + final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() + .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + } + + private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, + final HttpStatus expectedStatus, + final Throwable throwable) { + assertThat("Http Status", response.status(), equalTo(expectedStatus)); + assertThat("Http Response Throwable", throwable, is(nullValue())); + + final List headerKeys = response.headers() + .stream() + .map(Map.Entry::getKey) + .map(AsciiString::toString) + .collect(Collectors.toList()); + assertThat("Response Header Keys", headerKeys, not(contains("server"))); + } + + private byte[] createGZipCompressedPayload(final String payload) throws IOException { + // Create a GZip compressed request body + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { + gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); + } + return byteStream.toByteArray(); + } +} diff --git a/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSource_RetryInfoTest.java b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSource_RetryInfoTest.java new file mode 100644 index 0000000000..e1eb4cdecc --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/OTLPSource_RetryInfoTest.java @@ -0,0 +1,244 @@ +package org.opensearch.dataprepper.plugins.source.otlp; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.util.UUID; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.rpc.RetryInfo; +import com.linecorp.armeria.client.Clients; + +import io.grpc.Metadata; +import io.grpc.StatusRuntimeException; +import io.opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest; +import io.opentelemetry.proto.collector.logs.v1.LogsServiceGrpc; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.logs.v1.LogRecord; +import io.opentelemetry.proto.logs.v1.ResourceLogs; +import io.opentelemetry.proto.logs.v1.ScopeLogs; +import io.opentelemetry.proto.resource.v1.Resource; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + +import static org.opensearch.dataprepper.plugins.source.otlp.OTLPSourceConfig.DEFAULT_REQUEST_TIMEOUT; + +@ExtendWith(MockitoExtension.class) +public class OTLPSource_RetryInfoTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21893/"; + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), + Duration.ofMillis(2000)); + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTLPSourceConfig otlpSourceConfig; + + @Mock + private Buffer> buffer; + + private OTLPSource SOURCE; + + @BeforeEach + void beforeEach() throws Exception { + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); + + when(otlpSourceConfig.getPort()).thenReturn(21893); + when(otlpSourceConfig.isSsl()).thenReturn(false); + when(otlpSourceConfig.getRequestTimeoutInMillis()) + .thenReturn((int) Duration.ofSeconds(DEFAULT_REQUEST_TIMEOUT).toMillis()); + when(otlpSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(otlpSourceConfig.getThreadCount()).thenReturn(5); + when(otlpSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(otlpSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + + configureObjectUnderTest(); + SOURCE.start(buffer); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + PluginMetrics pluginMetrics = PluginMetrics.fromNames("otlp-source", "pipeline"); + PipelineDescription pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + + SOURCE = new OTLPSource(otlpSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + + @Test + public void metrics_failed_request_returns_minimal_delay_in_status() throws Exception { + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, + () -> client.export(createExportMetricsRequest())); + + RetryInfo retryInfo = extractRetryInfoFromStatusRuntimeException(statusRuntimeException); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); + } + + @Test + public void logs_failed_request_returns_minimal_delay_in_status() throws Exception { + final LogsServiceGrpc.LogsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, + () -> client.export(createExportLogsRequest())); + + RetryInfo retryInfo = extractRetryInfoFromStatusRuntimeException(statusRuntimeException); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); + } + + @Test + public void traces_failed_request_returns_minimal_delay_in_status() throws Exception { + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final StatusRuntimeException statusRuntimeException = assertThrows(StatusRuntimeException.class, + () -> client.export(createExportTraceRequest())); + + RetryInfo retryInfo = extractRetryInfoFromStatusRuntimeException(statusRuntimeException); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(100L)); + } + + @Test + public void metrics_failed_request_returns_extended_delay_in_status() throws Exception { + RetryInfo retryInfo = callMetricService3TimesAndReturnRetryInfo(); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); + } + + @Test + public void logs_failed_request_returns_extended_delay_in_status() throws Exception { + RetryInfo retryInfo = callLogsService3TimesAndReturnRetryInfo(); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); + } + + @Test + public void traces_failed_request_returns_extended_delay_in_status() throws Exception { + RetryInfo retryInfo = callTraceService3TimesAndReturnRetryInfo(); + assertThat(Duration.ofNanos(retryInfo.getRetryDelay().getNanos()).toMillis(), equalTo(200L)); + } + + private RetryInfo extractRetryInfoFromStatusRuntimeException(StatusRuntimeException e) + throws InvalidProtocolBufferException { + com.google.rpc.Status status = com.google.rpc.Status + .parseFrom(e.getTrailers() + .get(Metadata.Key.of("grpc-status-details-bin", Metadata.BINARY_BYTE_MARSHALLER))); + return RetryInfo.parseFrom(status.getDetails(0).getValue()); + } + + private RetryInfo callMetricService3TimesAndReturnRetryInfo() throws Exception { + StatusRuntimeException e = null; + for (int i = 0; i < 3; i++) { + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + } + + return extractRetryInfoFromStatusRuntimeException(e); + } + + private ExportMetricsServiceRequest createExportMetricsRequest() { + return ExportMetricsServiceRequest.newBuilder().build(); + } + + private RetryInfo callLogsService3TimesAndReturnRetryInfo() throws Exception { + StatusRuntimeException e = null; + for (int i = 0; i < 3; i++) { + final LogsServiceGrpc.LogsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(LogsServiceGrpc.LogsServiceBlockingStub.class); + e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportLogsRequest())); + } + + return extractRetryInfoFromStatusRuntimeException(e); + } + + private ExportLogsServiceRequest createExportLogsRequest() { + final Resource resource = Resource.newBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey("service.name") + .setValue(AnyValue.newBuilder().setStringValue("service").build())) + .build(); + + final ResourceLogs resourceLogs = ResourceLogs.newBuilder() + .addScopeLogs(ScopeLogs.newBuilder() + .addLogRecords(LogRecord.newBuilder().setSeverityNumberValue(1)) + .build()) + .setResource(resource) + .build(); + + return ExportLogsServiceRequest.newBuilder() + .addResourceLogs(resourceLogs) + .build(); + } + + private RetryInfo callTraceService3TimesAndReturnRetryInfo() throws Exception { + StatusRuntimeException e = null; + for (int i = 0; i < 3; i++) { + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + e = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); + } + + return extractRetryInfoFromStatusRuntimeException(e); + } + + private ExportTraceServiceRequest createExportTraceRequest() { + final Span testSpan = Span.newBuilder() + .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + ScopeSpans scopeSpan = ScopeSpans.newBuilder().addSpans(testSpan).build(); + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder().addScopeSpans(scopeSpan)).build(); + } +} diff --git a/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/certificate/CertificateProviderFactoryTest.java new file mode 100644 index 0000000000..ca46474dcf --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/java/org/opensearch/dataprepper/plugins/source/otlp/certificate/CertificateProviderFactoryTest.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otlp.certificate; + +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.hamcrest.core.IsInstanceOf; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.acm.ACMCertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.file.FileCertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.s3.S3CertificateProvider; +import org.opensearch.dataprepper.plugins.source.otlp.OTLPSourceConfig; + +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; + +@ExtendWith(MockitoExtension.class) +public class CertificateProviderFactoryTest { + private OTLPSourceConfig otlpSourceConfig; + private CertificateProviderFactory certificateProviderFactory; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + @Test + public void getCertificateProviderAcmProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("use_acm_certificate_for_ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("acm_certificate_arn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + certificateProviderFactory = new CertificateProviderFactory(otlpSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(ACMCertificateProvider.class)); + } + + @Test + public void getCertificateProviderS3ProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", true); + settingsMap.put("aws_region", "us-east-1"); + settingsMap.put("ssl_certificate_file", "s3://src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "s3://src/test/resources/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + otlpSourceConfig.validateAndInitializeCertAndKeyFileInS3(); + + certificateProviderFactory = new CertificateProviderFactory(otlpSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(S3CertificateProvider.class)); + } + + @Test + public void getCertificateProviderFileProviderSuccess() { + final Map settingsMap = new HashMap<>(); + settingsMap.put("ssl", true); + settingsMap.put("ssl_certificate_file", "src/test/resources/certificate/test_cert.crt"); + settingsMap.put("ssl_key_file", "src/test/resources/certificate/test_decrypted_key.key"); + + final PluginSetting pluginSetting = new PluginSetting(null, settingsMap); + pluginSetting.setPipelineName("pipeline"); + otlpSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), + OTLPSourceConfig.class); + + certificateProviderFactory = new CertificateProviderFactory(otlpSourceConfig); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + + assertThat(certificateProvider, IsInstanceOf.instanceOf(FileCertificateProvider.class)); + } +} diff --git a/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_cert.crt b/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_cert.crt new file mode 100644 index 0000000000..26c78d1411 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_cert.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICHTCCAYYCCQD4hqYeYDQZADANBgkqhkiG9w0BAQUFADBSMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCVFgxDzANBgNVBAcMBkF1c3RpbjEPMA0GA1UECgwGQW1hem9u +MRQwEgYDVQQLDAtEYXRhcHJlcHBlcjAgFw0yMTA2MjUxOTIzMTBaGA8yMTIxMDYw +MTE5MjMxMFowUjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAlRYMQ8wDQYDVQQHDAZB +dXN0aW4xDzANBgNVBAoMBkFtYXpvbjEUMBIGA1UECwwLRGF0YXByZXBwZXIwgZ8w +DQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAKrb3YhdKbQ5PtLHall10iLZC9ZdDVrq +HOvqVSM8NHlL8f82gJ3l0n9k7hYc5eKisutaS9eDTmJ+Dnn8xn/qPSKTIq9Wh+OZ +O+e9YEEpI/G4F9KpGULgMyRg9sJK0GlZdEt9o5GJNJIJUkptJU5eiLuE0IV+jyJo +Nvm8OE6EJPqxAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAjgnX5n/Tt7eo9uakIGAb +uBhvYdR8JqKXqF9rjFJ/MIK7FdQSF/gCdjnvBhzLlZFK/Nb6MGKoSKm5Lcr75LgC +FyhIwp3WlqQksiMFnOypYVY71vqDgj6UKdMaOBgthsYhngj8lC+wsVzWqQvkJ2Qg +/GAIzJwiZfXiaevQHRk79qI= +-----END CERTIFICATE----- diff --git a/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_decrypted_key.key b/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_decrypted_key.key new file mode 100644 index 0000000000..479b877131 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_decrypted_key.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXAIBAAKBgQCq292IXSm0OT7Sx2pZddIi2QvWXQ1a6hzr6lUjPDR5S/H/NoCd +5dJ/ZO4WHOXiorLrWkvXg05ifg55/MZ/6j0ikyKvVofjmTvnvWBBKSPxuBfSqRlC +4DMkYPbCStBpWXRLfaORiTSSCVJKbSVOXoi7hNCFfo8iaDb5vDhOhCT6sQIDAQAB +AoGANrrhFqpJDpr7vcb1ER0Fp/YArbT27zVo+EUC6puBb41dQlQyFOImcHpjLaAq +H1PgnjU5cBp2hGQ+vOK0rwrYc/HNl6vfh6N3NbDptMiuoBafRJA9JzYourAM09BU +zmXyr61Yn3KHzx1PRwWe37icX93oXP3P0qHb3dI1ZF4jG0ECQQDU5N/a7ogoz2zn +ZssD6FvUOUQDsdBWdXmhUvg+YdZrV44e4xk+FVzwEONoRktEYKz9MFXlsgNHr445 +KRguHWcJAkEAzXQkwOkN8WID1wrwoobUIMbZSGAZzofwkKXgTTnllnT1qOQXuRbS +aCMejFEymBBef4aXP6N4+va2FKW/MF34aQJAO2oMl1sOoOUSrZngepy0VAwPUUCk +thxe74jqQu6nGpn6zd/vQYZQw6bS8Fz90H1yic6dilcd1znFZWp0lxoZkQJBALeI +xoBycRsuFQIYasi1q3AwUtBd0Q/3zkZZeBtk2hzjFMUwJaUZpxKSNOrialD/ZnuD +jz+xWBTRKe0d98JMX+kCQCmsJEj/HYQAC1GamZ7JQWogRSRF2KTgTWRaDXDxy0d4 +yUQgwHB+HZLFcbi1JEK6eIixCsX8iifrrkteh+1npJ0= +-----END RSA PRIVATE KEY----- diff --git a/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_encrypted_key.key b/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_encrypted_key.key new file mode 100644 index 0000000000..285efc8d82 --- /dev/null +++ b/data-prepper-plugins/otlp-source/src/test/resources/certificate/test_encrypted_key.key @@ -0,0 +1,17 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIICojAcBgoqhkiG9w0BDAEDMA4ECAd2FKZw2oGwAgIIAASCAoDTgiaXkazaotc7 +SxQK3bX34sEvdkLXg/O4ZpHTb0f4gLPxNhDe7ZPKrAS2TdywpSHT0189MVl+PIvw +4YQDaGVHL1SM5ukJu+PQkfQAMigdCJ+bUsG6hkrUDC74qYhHZHj1yVGavL6I4KHT +Ixh9IV2GMRS4m6HGJ17nYsdiTFFNdK++WcTMxbVWv3SNdKGZG79T32pjMxuIUPWr +3dB+ZXM+FSqwlBLZxPvvjlP6ETw7QXrlBHcQh1tHSh10bM+q0c5CktZoXLwpc+gv +ZsGXzpVjqFrAw4Vw0ikJl1mUCoGOtqqP0P6QWwbIJZBxNoO0MvWcXW+U3AGNFFze +nMR8UTXdga2l1Lx7pokQkWUpp48SDRjDx/RdZTRXCgtRcKuBcm0x2lxNILdwOzjJ +5GlKMvvc2OXXTnYqSCTqdfbuR3XBYmWgFki92D6JnVIYq+QJr5qj8IJDJ7mADQ1i +Za6PEJnrT641fLeSKRq7QiTydMQ3JXa9DFqUPwdZPPHLr/hC19sWHrq7gxvhkcLI +wrTtTIi8/iV4IVaiHk7YU83IM6sGExabQ3BRXcHMr+7i1vVxtEsFNC6ycTfJ8gpJ +YsnpXUQe912l5sk7GRSP1InNRF7kzMD0QeOAQ0UVfmsbHOPSXvD7fXkJWIb6N+zW +qCQYRmBwc7Bz2KZein5MLsMeNz2AWj/DcA2fMC+4+QtI0nF5BFtaR0V0npWhsbPu +3rj+AXipnvVhDIkfl8495j7ybCBj7HAZk8Ux8GmiZ3PGFO1C7XCQaLPWJ4Aw4Kb3 +EUqtVtpbwsCov5PDmMDXgz8qOxWMdQsP/dPF1HnVAg7SoFG9xA4nHdZ2WAFZwYtf +rRxEd7br +-----END ENCRYPTED PRIVATE KEY----- diff --git a/settings.gradle b/settings.gradle index 319fdc467f..8b291856a1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -121,6 +121,7 @@ include 'data-prepper-plugins:otel-trace-source' include 'data-prepper-plugins:otel-metrics-source' include 'data-prepper-plugins:otel-metrics-raw-processor' include 'data-prepper-plugins:otel-logs-source' +include 'data-prepper-plugins:otlp-source' include 'data-prepper-plugins:blocking-buffer' include 'data-prepper-plugins:http-source' include 'data-prepper-plugins:drop-events-processor' @@ -200,3 +201,4 @@ include 'data-prepper-plugins:saas-source-plugins:atlassian-commons' include 'data-prepper-plugins:saas-source-plugins:crowdstrike-source' include 'data-prepper-plugins:saas-source-plugins:microsoft-office365-source' include 'data-prepper-plugins:otlp-sink' +