diff --git a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigFixture.java b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigFixture.java index 115c26543d..0e8f509af9 100644 --- a/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigFixture.java +++ b/data-prepper-plugins/otel-logs-source/src/test/java/org/opensearch/dataprepper/plugins/source/otellogs/OtelLogsSourceConfigFixture.java @@ -14,7 +14,6 @@ import static org.opensearch.dataprepper.plugins.source.otellogs.OTelLogsSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; import static org.opensearch.dataprepper.plugins.source.otellogs.OtelLogsSourceConfigTestData.BASIC_AUTH_PASSWORD; import static org.opensearch.dataprepper.plugins.source.otellogs.OtelLogsSourceConfigTestData.BASIC_AUTH_USERNAME; -import static org.opensearch.dataprepper.plugins.source.otellogs.OtelLogsSourceConfigTestData.CONFIG_HTTP_PATH; import java.util.Map; @@ -42,7 +41,6 @@ public static OTelLogsSourceConfig createDefaultConfig() { public static OTelLogsSourceConfig.OTelLogsSourceConfigBuilder createDefaultConfigBuilder() { return OTelLogsSourceConfig.builder() .healthCheck(true) - .httpPath(CONFIG_HTTP_PATH) .port(DEFAULT_PORT) .enableUnframedRequests(false) .ssl(false) diff --git a/data-prepper-plugins/otel-metrics-source/build.gradle b/data-prepper-plugins/otel-metrics-source/build.gradle index d71d31f60d..814498f4f0 100644 --- a/data-prepper-plugins/otel-metrics-source/build.gradle +++ b/data-prepper-plugins/otel-metrics-source/build.gradle @@ -15,7 +15,6 @@ dependencies { implementation project(':data-prepper-plugins:armeria-common') implementation project(':data-prepper-plugins:otel-proto-common') implementation project(':data-prepper-plugins:http-common') - implementation project(':data-prepper-plugins:http-source-common' ) testImplementation project(':data-prepper-api').sourceSets.test.output implementation libs.opentelemetry.proto implementation libs.commons.io @@ -35,8 +34,6 @@ dependencies { testImplementation 'org.assertj:assertj-core:3.27.7' testImplementation libs.commons.io testImplementation 'org.skyscreamer:jsonassert:1.5.3' - implementation 'org.projectlombok:lombok:1.18.26' - annotationProcessor 'org.projectlombok:lombok:1.18.26' } jacocoTestCoverageVerification { diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/ConvertConfiguration.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/ConvertConfiguration.java new file mode 100644 index 0000000000..3922a10572 --- /dev/null +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/ConvertConfiguration.java @@ -0,0 +1,27 @@ +package org.opensearch.dataprepper.plugins.source.otelmetrics; + +import org.opensearch.dataprepper.plugins.server.ServerConfiguration; + +public class ConvertConfiguration { + + public static ServerConfiguration convertConfiguration(final OTelMetricsSourceConfig oTelMetricsSourceConfig) { + ServerConfiguration serverConfiguration = new ServerConfiguration(); + serverConfiguration.setPath(oTelMetricsSourceConfig.getPath()); + serverConfiguration.setHealthCheck(oTelMetricsSourceConfig.hasHealthCheck()); + serverConfiguration.setProtoReflectionService(oTelMetricsSourceConfig.hasProtoReflectionService()); + serverConfiguration.setRequestTimeoutInMillis(oTelMetricsSourceConfig.getRequestTimeoutInMillis()); + serverConfiguration.setEnableUnframedRequests(oTelMetricsSourceConfig.enableUnframedRequests()); + serverConfiguration.setCompression(oTelMetricsSourceConfig.getCompression()); + serverConfiguration.setAuthentication(oTelMetricsSourceConfig.getAuthentication()); + serverConfiguration.setSsl(oTelMetricsSourceConfig.isSsl()); + serverConfiguration.setUnauthenticatedHealthCheck(oTelMetricsSourceConfig.isUnauthenticatedHealthCheck()); + serverConfiguration.setUseAcmCertForSSL(oTelMetricsSourceConfig.useAcmCertForSSL()); + serverConfiguration.setMaxRequestLength(oTelMetricsSourceConfig.getMaxRequestLength()); + serverConfiguration.setPort(oTelMetricsSourceConfig.getPort()); + serverConfiguration.setRetryInfo(oTelMetricsSourceConfig.getRetryInfo()); + serverConfiguration.setThreadCount(oTelMetricsSourceConfig.getThreadCount()); + serverConfiguration.setMaxConnectionCount(oTelMetricsSourceConfig.getMaxConnectionCount()); + + return serverConfiguration; + } +} diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcService.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcService.java index 99d48be468..1b126e613b 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcService.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcService.java @@ -1,11 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics; diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java index cda5e26b61..a48d80280a 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource.java @@ -1,32 +1,16 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics; -import static org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME; - -import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; -import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; -import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.encoding.DecodingService; -import com.linecorp.armeria.server.grpc.GrpcService; -import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; -import com.linecorp.armeria.server.healthcheck.HealthCheckService; -import com.linecorp.armeria.server.throttling.ThrottlingService; - -import org.opensearch.dataprepper.GrpcRequestExceptionHandler; -import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; +import io.grpc.MethodDescriptor; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; -import org.opensearch.dataprepper.http.LogThrottlingRejectHandler; -import org.opensearch.dataprepper.http.LogThrottlingStrategy; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -40,58 +24,33 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.model.Certificate; -import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.plugins.otel.codec.OTelMetricDecoder; import org.opensearch.dataprepper.plugins.otel.codec.OTelOutputFormat; -import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoOpensearchCodec; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoStandardCodec; import org.opensearch.dataprepper.plugins.server.CreateServer; -import org.opensearch.dataprepper.plugins.server.HealthGrpcService; -import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; +import org.opensearch.dataprepper.plugins.server.ServerConfiguration; import org.opensearch.dataprepper.plugins.source.otelmetrics.certificate.CertificateProviderFactory; -import org.opensearch.dataprepper.plugins.source.otelmetrics.http.ArmeriaHttpService; -import org.opensearch.dataprepper.plugins.source.otelmetrics.http.HttpExceptionHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -import io.grpc.ServerInterceptor; -import io.grpc.ServerInterceptors; -import io.grpc.protobuf.services.ProtoReflectionService; -import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; @DataPrepperPlugin(name = "otel_metrics_source", pluginType = Source.class, pluginConfigurationType = OTelMetricsSourceConfig.class) public class OTelMetricsSource implements Source> { + private static final String PLUGIN_NAME = "otel_metrics_source"; private static final Logger LOG = LoggerFactory.getLogger(OTelMetricsSource.class); static final String SERVER_CONNECTIONS = "serverConnections"; private final OTelMetricsSourceConfig oTelMetricsSourceConfig; private final String pipelineName; private final PluginMetrics pluginMetrics; + private final GrpcAuthenticationProvider authenticationProvider; private final CertificateProviderFactory certificateProviderFactory; - private final PluginFactory pluginFactory; private Server server; private final ByteDecoder byteDecoder; - private static final int MAX_PENDING_REQUESTS = 1024; - private static final int BUFFER_WRITE_TIMEOUT_IN_MS = 100; - private static final String HTTP_HEALTH_CHECK_PATH = "/health"; - static final String REGEX_HEALTH = "regex:^/(?!health$).*$"; - @DataPrepperPluginConstructor public OTelMetricsSource(final OTelMetricsSourceConfig oTelMetricsSourceConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory, final PipelineDescription pipelineDescription) { @@ -99,17 +58,14 @@ public OTelMetricsSource(final OTelMetricsSourceConfig oTelMetricsSourceConfig, } // accessible only in the same package for unit test - OTelMetricsSource(final OTelMetricsSourceConfig oTelMetricsSourceConfig, - final PluginMetrics pluginMetrics, - final PluginFactory pluginFactory, - final CertificateProviderFactory certificateProviderFactory, - final PipelineDescription pipelineDescription) { + OTelMetricsSource(final OTelMetricsSourceConfig oTelMetricsSourceConfig, final PluginMetrics pluginMetrics, final PluginFactory pluginFactory, + final CertificateProviderFactory certificateProviderFactory, final PipelineDescription pipelineDescription) { oTelMetricsSourceConfig.validateAndInitializeCertAndKeyFileInS3(); this.oTelMetricsSourceConfig = oTelMetricsSourceConfig; this.pluginMetrics = pluginMetrics; this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); - this.pluginFactory = pluginFactory; + this.authenticationProvider = createAuthenticationProvider(pluginFactory); this.byteDecoder = new OTelMetricDecoder(oTelMetricsSourceConfig.getOutputFormat()); } @@ -125,19 +81,24 @@ public void start(Buffer> buffer) { } if (server == null) { - final ServerBuilder serverBuilder = Server.builder(); - - configureHeadersAndHealthCheck(serverBuilder); - configureTLS(serverBuilder); - configureAuthentication(serverBuilder); - final ScheduledThreadPoolExecutor executor = configureTaskExecutor(serverBuilder); - - configureGrpcService(serverBuilder, buffer); - if (oTelMetricsSourceConfig.getHttpPath() != null) { - configureHttpService(serverBuilder, buffer, executor.getQueue()); + final OTelMetricsGrpcService oTelMetricsGrpcService = new OTelMetricsGrpcService( + (int) (oTelMetricsSourceConfig.getRequestTimeoutInMillis() * 0.8), + oTelMetricsSourceConfig.getOutputFormat() == OTelOutputFormat.OPENSEARCH ? new OTelProtoOpensearchCodec.OTelProtoDecoder() : new OTelProtoStandardCodec.OTelProtoDecoder(), + buffer, + oTelMetricsSourceConfig.getBufferPartitionKeys(), + pluginMetrics, + null + ); + + ServerConfiguration serverConfiguration = ConvertConfiguration.convertConfiguration(oTelMetricsSourceConfig); + CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, PLUGIN_NAME, pipelineName); + CertificateProvider certificateProvider = null; + if (oTelMetricsSourceConfig.isSsl() || oTelMetricsSourceConfig.useAcmCertForSSL()) { + certificateProvider = certificateProviderFactory.getCertificateProvider(); } + final MethodDescriptor methodDescriptor = MetricsServiceGrpc.getExportMethod(); + server = createServer.createGRPCServer(authenticationProvider, oTelMetricsGrpcService, certificateProvider, methodDescriptor); - server = serverBuilder.build(); pluginMetrics.gauge(SERVER_CONNECTIONS, server, Server::numConnections); } try { @@ -155,76 +116,27 @@ public void start(Buffer> buffer) { LOG.info("Started otel_metrics_source..."); } - private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { - serverBuilder.disableServerHeader(); - - if ((oTelMetricsSourceConfig.isEnableUnframedRequests() || oTelMetricsSourceConfig.getHttpPath() != null) - && oTelMetricsSourceConfig.isHealthCheck()) { - LOG.info("HTTP source health check is enabled for metrics source"); - serverBuilder.service(HTTP_HEALTH_CHECK_PATH, HealthCheckService.builder().longPolling(0).build()); - } - - serverBuilder.maxNumConnections(oTelMetricsSourceConfig.getMaxConnectionCount()); - serverBuilder.requestTimeout(Duration.ofMillis(oTelMetricsSourceConfig.getRequestTimeoutInMillis())); - - if (oTelMetricsSourceConfig.getMaxRequestLength() != null) { - serverBuilder.maxRequestLength(oTelMetricsSourceConfig.getMaxRequestLength().getBytes()); - } - } - - private void configureTLS(ServerBuilder serverBuilder) { - if (oTelMetricsSourceConfig.isSsl()) { - LOG.info("Creating metrics http source with SSL/TLS enabled."); - final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); - final Certificate certificate = certificateProvider.getCertificate(); - // TODO: enable encrypted key with password - serverBuilder.https(oTelMetricsSourceConfig.getPort()).tls( - new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), - new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8))); - } else { - LOG.warn("Creating otlp metrics source without SSL/TLS. This is not secure."); - LOG.warn("In order to set up TLS for the otlp metrics source, go here: https://docs.opensearch.org/latest/data-prepper/pipelines/configuration/sources/otel-metrics-source/#ssl"); - serverBuilder.http(oTelMetricsSourceConfig.getPort()); - } - } - - private void configureAuthentication(ServerBuilder serverBuilder) { - if (oTelMetricsSourceConfig.getAuthentication() != null) { - final Optional> optionalAuthDecorator = - createHttpAuthentication() - .flatMap(ArmeriaHttpAuthenticationProvider::getAuthenticationDecorator); - - if (oTelMetricsSourceConfig.isUnauthenticatedHealthCheck()) { - optionalAuthDecorator.ifPresent(authDecorator -> - serverBuilder.decorator(REGEX_HEALTH, authDecorator)); - } else { - optionalAuthDecorator.ifPresent(serverBuilder::decorator); + @Override + public void stop() { + if (server != null) { + try { + server.stop().get(); + } catch (ExecutionException ex) { + if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { + throw (RuntimeException) ex.getCause(); + } else { + throw new RuntimeException(ex); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new RuntimeException(ex); } } + LOG.info("Stopped otel_metrics_source."); } - private ScheduledThreadPoolExecutor configureTaskExecutor(ServerBuilder serverBuilder) { - final ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(oTelMetricsSourceConfig.getThreadCount()); - serverBuilder.blockingTaskExecutor(executor, true); - return executor; - } - - private Optional createHttpAuthentication() { - if (oTelMetricsSourceConfig.getAuthentication() == null || oTelMetricsSourceConfig.getAuthentication().getPluginName().equals(UNAUTHENTICATED_PLUGIN_NAME)) { - LOG.warn("Creating otel_metrics_source http service without authentication. This is not secure."); - LOG.warn("In order to set up Http Basic authentication for the otel-metrics-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-metrics-source#authentication-configurations"); - return Optional.empty(); - } else { - return Optional.of(createHttpAuthenticationProvider(oTelMetricsSourceConfig.getAuthentication())); - } - } - - private ArmeriaHttpAuthenticationProvider createHttpAuthenticationProvider(final PluginModel authenticationConfiguration) { - Map pluginSettings = authenticationConfiguration.getPluginSettings(); - return pluginFactory.loadPlugin(ArmeriaHttpAuthenticationProvider.class, new PluginSetting(authenticationConfiguration.getPluginName(), pluginSettings)); - } - private GrpcAuthenticationProvider createGrpcAuthenticationProvider(final PluginFactory pluginFactory) { + private GrpcAuthenticationProvider createAuthenticationProvider(final PluginFactory pluginFactory) { final PluginModel authenticationConfiguration = oTelMetricsSourceConfig.getAuthentication(); if (authenticationConfiguration == null || authenticationConfiguration.getPluginName().equals(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { @@ -241,116 +153,4 @@ private GrpcAuthenticationProvider createGrpcAuthenticationProvider(final Plugin authenticationPluginSetting.setPipelineName(pipelineName); return pluginFactory.loadPlugin(GrpcAuthenticationProvider.class, authenticationPluginSetting); } - - private void configureHttpService(ServerBuilder serverBuilder, Buffer> buffer, BlockingQueue blockingQueue) { - final String path = oTelMetricsSourceConfig.getHttpPath().replace("${pipelineName}", pipelineName); - LOG.info("Configuring HTTP service under {} ", path); - - final ArmeriaHttpService armeriaHttpService = new ArmeriaHttpService(buffer, pluginMetrics, BUFFER_WRITE_TIMEOUT_IN_MS, createOtelProtoDecoder()); - final RetryInfoConfig retryInfo = oTelMetricsSourceConfig.getRetryInfo() != null ? oTelMetricsSourceConfig.getRetryInfo() : new RetryInfoConfig(); - final HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); - - final int maxPendingRequests = MAX_PENDING_REQUESTS; - final LogThrottlingStrategy logThrottlingStrategy = new LogThrottlingStrategy(maxPendingRequests, blockingQueue); - final LogThrottlingRejectHandler logThrottlingRejectHandler = new LogThrottlingRejectHandler(maxPendingRequests, pluginMetrics); - serverBuilder.decorator(path, ThrottlingService.newDecorator(logThrottlingStrategy, logThrottlingRejectHandler)); - - if (CompressionOption.NONE.equals(oTelMetricsSourceConfig.getCompression())) { - serverBuilder.annotatedService(path, armeriaHttpService, httpExceptionHandler); - } else { - serverBuilder.annotatedService( - path, - armeriaHttpService, - DecodingService.newDecorator(), - httpExceptionHandler); - } - - } - - private OTelProtoCodec.OTelProtoDecoder createOtelProtoDecoder() { - return oTelMetricsSourceConfig.getOutputFormat() == OTelOutputFormat.OPENSEARCH ? new OTelProtoOpensearchCodec.OTelProtoDecoder() : new OTelProtoStandardCodec.OTelProtoDecoder(); - } - - private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { - LOG.info("Configuring gRPC metrics service"); - - final GrpcServiceBuilder grpcServiceBuilder = GrpcService - .builder() - .useClientTimeoutHeader(false) - .useBlockingTaskExecutor(true) - .exceptionHandler(createGrpcExceptionHandler(oTelMetricsSourceConfig)); - final OTelMetricsGrpcService oTelmetricsGrpcService = new OTelMetricsGrpcService( - (int) (oTelMetricsSourceConfig.getRequestTimeoutInMillis() * 0.8), - createOtelProtoDecoder(), - buffer, - oTelMetricsSourceConfig.getBufferPartitionKeys(), - pluginMetrics, - null - ); - GrpcAuthenticationProvider authProvider = createGrpcAuthenticationProvider(pluginFactory); - - final List interceptors = new ArrayList<>(); - if (authProvider.getAuthenticationInterceptor() != null) { - interceptors.add(authProvider.getAuthenticationInterceptor()); - } - - if (oTelMetricsSourceConfig.isEnableUnframedRequests()) { - grpcServiceBuilder.enableUnframedRequests(true); - } - - final CreateServer.GRPCServiceConfig grpcServiceConfig = new CreateServer.GRPCServiceConfig<>(oTelmetricsGrpcService); - if (oTelMetricsSourceConfig.getPath() != null) { - final String path = oTelMetricsSourceConfig.getPath().replace("${pipelineName}", pipelineName); - LOG.info("custom gRPC metrics path: {} ", path); - grpcServiceBuilder.addService( - path, - ServerInterceptors.intercept(grpcServiceConfig.getService(), interceptors), - MetricsServiceGrpc.getExportMethod()); - } else { - grpcServiceBuilder.addService(ServerInterceptors.intercept(grpcServiceConfig.getService(), interceptors)); - } - - if (oTelMetricsSourceConfig.isHealthCheck()) { - LOG.info("Health check for gRPC metrics service is enabled"); - grpcServiceBuilder.addService(new HealthGrpcService()); - } - - if (oTelMetricsSourceConfig.isProtoReflectionService()) { - LOG.info("Proto reflection service for metrics source is enabled"); - grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); - } - - if (CompressionOption.NONE.equals(oTelMetricsSourceConfig.getCompression())) { - serverBuilder.service(grpcServiceBuilder.build()); - } else { - serverBuilder.service(grpcServiceBuilder.build(), DecodingService.newDecorator()); - } - } - - private GrpcExceptionHandlerFunction createGrpcExceptionHandler(OTelMetricsSourceConfig config) { - RetryInfoConfig retryInfo = config.getRetryInfo() != null - ? config.getRetryInfo() - : new RetryInfoConfig(); - - return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); - } - - @Override - public void stop() { - if (server != null) { - try { - server.stop().get(); - } catch (ExecutionException ex) { - if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { - throw (RuntimeException) ex.getCause(); - } else { - throw new RuntimeException(ex); - } - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - throw new RuntimeException(ex); - } - } - LOG.info("Stopped otel_metrics_source."); - } } diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java index be86528d4e..d6f95927ea 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfig.java @@ -1,11 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics; @@ -13,11 +8,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Size; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Getter; -import lombok.NoArgsConstructor; - import org.apache.commons.lang3.StringUtils; import org.opensearch.dataprepper.model.types.ByteCount; import org.opensearch.dataprepper.plugins.codec.CompressionOption; @@ -27,11 +17,6 @@ import java.util.Set; - -@Builder -@AllArgsConstructor -@NoArgsConstructor -@Getter public class OTelMetricsSourceConfig { static final String REQUEST_TIMEOUT = "request_timeout"; static final String PORT = "port"; @@ -67,9 +52,6 @@ public class OTelMetricsSourceConfig { static final String UNAUTHENTICATED_HEALTH_CHECK = "unauthenticated_health_check"; private static final String NAME_KEY = "name"; private static final String SERVICE_NAME_KEY = "service_name"; - static final String HTTP_PATH = "http_path"; - static final String AUTHENTICATION = "authentication"; - static final String MAX_REQUEST_LENGTH = "max_request_length"; @JsonProperty(REQUEST_TIMEOUT) private int requestTimeoutInMillis = DEFAULT_REQUEST_TIMEOUT_MS; @@ -129,7 +111,7 @@ public class OTelMetricsSourceConfig { @JsonProperty(MAX_CONNECTION_COUNT) private int maxConnectionCount = DEFAULT_MAX_CONNECTION_COUNT; - @JsonProperty(AUTHENTICATION) + @JsonProperty("authentication") private PluginModel authentication; @JsonProperty(UNAUTHENTICATED_HEALTH_CHECK) @@ -138,16 +120,12 @@ public class OTelMetricsSourceConfig { @JsonProperty(COMPRESSION) private CompressionOption compression = CompressionOption.NONE; - @JsonProperty(MAX_REQUEST_LENGTH) + @JsonProperty("max_request_length") private ByteCount maxRequestLength; @JsonProperty(RETRY_INFO) private RetryInfoConfig retryInfo; - @JsonProperty(HTTP_PATH) - @Size(min = 1, message = "path length should be at least 1") - private String httpPath; - @AssertTrue(message = "buffer_partition_keys only supports 'name' and 'service_name'. 'name' is mandatory") boolean isBufferKeysValid() { if (bufferPartitionKeys == null) { @@ -195,8 +173,106 @@ private boolean isSSLCertificateLocatedInS3() { sslKeyFile.toLowerCase().startsWith(S3_PREFIX); } + public int getRequestTimeoutInMillis() { + return requestTimeoutInMillis; + } + + public int getPort() { + return port; + } + + public String getPath() { + return path; + } + + public boolean hasHealthCheck() { + return healthCheck; + } + + public Set getBufferPartitionKeys() { + return bufferPartitionKeys; + } + public boolean enableHttpHealthCheck() { - return isEnableUnframedRequests() && isHealthCheck(); + return enableUnframedRequests() && hasHealthCheck(); + } + + public boolean hasProtoReflectionService() { + return protoReflectionService; + } + + public boolean enableUnframedRequests() { + return enableUnframedRequests; + } + + public boolean isSsl() { + return ssl; + } + + public OTelOutputFormat getOutputFormat() { + return outputFormat; + } + + public boolean useAcmCertForSSL() { + return useAcmCertForSSL; + } + + public long getAcmCertIssueTimeOutMillis() { + return acmCertIssueTimeOutMillis; + } + + public String getSslKeyCertChainFile() { + return sslKeyCertChainFile; + } + + public String getSslKeyFile() { + return sslKeyFile; + } + + public String getAcmCertificateArn() { + return acmCertificateArn; + } + + public String getAcmPrivateKeyPassword() { + return acmPrivateKeyPassword; + } + + public boolean isSslCertAndKeyFileInS3() { + return sslCertAndKeyFileInS3; + } + + public String getAwsRegion() { + return awsRegion; + } + + public int getThreadCount() { + return threadCount; + } + + public int getMaxConnectionCount() { + return maxConnectionCount; + } + + public PluginModel getAuthentication() { return authentication; } + + public boolean isUnauthenticatedHealthCheck() { + return unauthenticatedHealthCheck; + } + + public CompressionOption getCompression() { + return compression; + } + + public ByteCount getMaxRequestLength() { + return maxRequestLength; + } + + public RetryInfoConfig getRetryInfo() { + return retryInfo; + } + + public void setRetryInfo(RetryInfoConfig retryInfo) { + this.retryInfo = retryInfo; } } diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactory.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactory.java index 051278ab50..fd29456e71 100644 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactory.java +++ b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactory.java @@ -42,7 +42,7 @@ public CertificateProviderFactory(final OTelMetricsSourceConfig oTelMetricsSourc public CertificateProvider getCertificateProvider() { // ACM Cert for SSL takes preference - if (oTelMetricsSourceConfig.isUseAcmCertForSSL()) { + if (oTelMetricsSourceConfig.useAcmCertForSSL()) { LOG.info("Using ACM certificate and private key for SSL/TLS."); final AwsCredentialsProvider credentialsProvider = AwsCredentialsProviderChain.builder() .addCredentialsProvider(DefaultCredentialsProvider.create()).build(); diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/http/ArmeriaHttpService.java deleted file mode 100644 index a254283c61..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/http/ArmeriaHttpService.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.otelmetrics.http; - -import java.time.Instant; -import java.util.Collection; - -import org.opensearch.dataprepper.exceptions.BadRequestException; -import org.opensearch.dataprepper.exceptions.BufferWriteException; -import org.opensearch.dataprepper.logging.DataPrepperMarkers; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.metric.Metric; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.linecorp.armeria.server.ServiceRequestContext; -import com.linecorp.armeria.server.annotation.ConsumesJson; -import com.linecorp.armeria.server.annotation.ConsumesProtobuf; -import com.linecorp.armeria.server.annotation.Post; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.DistributionSummary; -import io.micrometer.core.instrument.Timer; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; - -public class ArmeriaHttpService { - private static final Logger LOG = LoggerFactory.getLogger(ArmeriaHttpService.class); - - public static final String REQUESTS_RECEIVED = "requestsReceived"; - public static final String SUCCESS_REQUESTS = "successRequests"; - public static final String PAYLOAD_SIZE = "payloadSize"; - public static final String REQUEST_PROCESS_DURATION = "requestProcessDuration"; - - private final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder; - private final Buffer> buffer; - - private final int bufferWriteTimeoutInMillis; - - private final Counter requestsReceivedCounter; - private final Counter successRequestsCounter; - private final DistributionSummary payloadSizeSummary; - private final Timer requestProcessDuration; - - public ArmeriaHttpService(Buffer> buffer, - final PluginMetrics pluginMetrics, - final int bufferWriteTimeoutInMillis, - final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder ) { - this.buffer = buffer; - this.oTelProtoDecoder = oTelProtoDecoder; - this.bufferWriteTimeoutInMillis = bufferWriteTimeoutInMillis; - - requestsReceivedCounter = pluginMetrics.counter(REQUESTS_RECEIVED); - successRequestsCounter = pluginMetrics.counter(SUCCESS_REQUESTS); - payloadSizeSummary = pluginMetrics.summary(PAYLOAD_SIZE); - requestProcessDuration = pluginMetrics.timer(REQUEST_PROCESS_DURATION); - } - - // no path provided. Will be set by config. - @Post("") - @ConsumesJson - @ConsumesProtobuf - public ExportMetricsServiceResponse exportMetrics(ExportMetricsServiceRequest request) { - requestsReceivedCounter.increment(); - payloadSizeSummary.record(request.getSerializedSize()); - - requestProcessDuration.record(() -> processRequest(request)); - - return ExportMetricsServiceResponse.newBuilder().build(); - } - - private void processRequest(final ExportMetricsServiceRequest request) { - final Collection> metrics; - - try { - metrics = oTelProtoDecoder.parseExportMetricsServiceRequest(request, Instant.now()); - } catch (Exception e) { - LOG.warn(DataPrepperMarkers.SENSITIVE, "Failed to parse the request with error {}. Request body: {}", e, request); - throw new BadRequestException(e.getMessage(), e); - } - - try { - if (buffer.isByteBuffer()) { - buffer.writeBytes(request.toByteArray(), null, bufferWriteTimeoutInMillis); - } else { - buffer.writeAll(metrics, bufferWriteTimeoutInMillis); - } - } catch (Exception e) { - if (ServiceRequestContext.current().isTimedOut()) { - LOG.warn("Exception writing to buffer but request already timed out.", e); - return; - } - - LOG.error("Failed to write the request of size {} due to:", request.toString().length(), e); - throw new BufferWriteException(e.getMessage(), e); - } - - if (ServiceRequestContext.current().isTimedOut()) { - LOG.warn("Buffer write completed successfully but request already timed out."); - return; - } - - successRequestsCounter.increment(); - } -} diff --git a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/http/HttpExceptionHandler.java b/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/http/HttpExceptionHandler.java deleted file mode 100644 index 24a1db5cee..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/main/java/org/opensearch/dataprepper/plugins/source/otelmetrics/http/HttpExceptionHandler.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.otelmetrics.http; - - -import java.time.Duration; -import java.util.concurrent.TimeoutException; - -import org.opensearch.dataprepper.RetryInfoCalculator; -import org.opensearch.dataprepper.exceptions.BadRequestException; -import org.opensearch.dataprepper.exceptions.BufferWriteException; -import org.opensearch.dataprepper.exceptions.RequestCancelledException; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.buffer.SizeOverflowException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.protobuf.Any; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.google.rpc.RetryInfo; -import com.linecorp.armeria.common.ContentTooLargeException; -import com.linecorp.armeria.common.HttpRequest; -import com.linecorp.armeria.common.HttpResponse; -import com.linecorp.armeria.common.HttpStatus; -import com.linecorp.armeria.common.MediaType; -import com.linecorp.armeria.server.HttpStatusException; -import com.linecorp.armeria.server.RequestTimeoutException; -import com.linecorp.armeria.server.ServiceRequestContext; -import com.linecorp.armeria.server.annotation.ExceptionHandlerFunction; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.micrometer.core.instrument.Counter; - -public class HttpExceptionHandler implements ExceptionHandlerFunction { - private static final Logger LOG = LoggerFactory.getLogger(HttpExceptionHandler.class); - - static final String ARMERIA_REQUEST_TIMEOUT_MESSAGE = "Timeout waiting for request to be served. This is usually due to the buffer being full."; - public static final String REQUEST_TIMEOUTS = "requestTimeouts"; - public static final String BAD_REQUESTS = "badRequests"; - public static final String REQUESTS_TOO_LARGE = "requestsTooLarge"; - public static final String INTERNAL_SERVER_ERROR = "internalServerError"; - - private final Counter requestTimeoutsCounter; - private final Counter badRequestsCounter; - private final Counter requestsTooLargeCounter; - private final Counter internalServerErrorCounter; - private final RetryInfoCalculator retryInfoCalculator; - - public HttpExceptionHandler(final PluginMetrics pluginMetrics, Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { - requestTimeoutsCounter = pluginMetrics.counter(REQUEST_TIMEOUTS); - badRequestsCounter = pluginMetrics.counter(BAD_REQUESTS); - requestsTooLargeCounter = pluginMetrics.counter(REQUESTS_TOO_LARGE); - internalServerErrorCounter = pluginMetrics.counter(INTERNAL_SERVER_ERROR); - this.retryInfoCalculator = new RetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); - } - - @Override - public HttpResponse handleException(final ServiceRequestContext ctx, - final HttpRequest req, - final Throwable e) { - final Throwable exceptionCause; - if (e instanceof BufferWriteException) { - exceptionCause = e.getCause(); - } else if (e instanceof HttpStatusException) { - exceptionCause = e.getCause(); - } else { - exceptionCause = e; - } - - StatusHolder statusHolder = createStatus(exceptionCause); - - try { - JsonFormat.TypeRegistry typeRegistry = JsonFormat.TypeRegistry.newBuilder() - .add(RetryInfo.getDescriptor()) - .build(); - - JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(typeRegistry); - return HttpResponse.of(statusHolder.getHttpStatus(), MediaType.JSON, printer.print(statusHolder.getStatus())); - } catch (InvalidProtocolBufferException ipbe) { - throw new RuntimeException(ipbe); - } - } - - private StatusHolder createStatus(Throwable e) { - if (e instanceof RequestTimeoutException || e instanceof TimeoutException) { - requestTimeoutsCounter.increment(); - return new StatusHolder(createStatus(e, Status.Code.RESOURCE_EXHAUSTED), createHttpStatusFromProtoBufStatus(Status.Code.RESOURCE_EXHAUSTED)); - } else if (e instanceof SizeOverflowException || e instanceof ContentTooLargeException) { - requestsTooLargeCounter.increment(); - return new StatusHolder(createStatus(e, Status.Code.RESOURCE_EXHAUSTED), createHttpStatusFromProtoBufStatus(Status.Code.RESOURCE_EXHAUSTED)); - } else if (e instanceof BadRequestException) { - badRequestsCounter.increment(); - return new StatusHolder(createStatus(e, Status.Code.INVALID_ARGUMENT), createHttpStatusFromProtoBufStatus(Status.Code.INVALID_ARGUMENT)); - } else if ((e instanceof StatusRuntimeException) && (e.getMessage().contains("Invalid protobuf byte sequence") || e.getMessage().contains("Can't decode compressed frame"))) { - badRequestsCounter.increment(); - return new StatusHolder(createStatus(e, Status.Code.INVALID_ARGUMENT), createHttpStatusFromProtoBufStatus(Status.Code.INVALID_ARGUMENT)); - } else if (e instanceof RequestCancelledException) { - requestTimeoutsCounter.increment(); - return new StatusHolder(createStatus(e, Status.Code.CANCELLED), createHttpStatusFromProtoBufStatus(Status.Code.CANCELLED)); - } else { - LOG.error("Unexpected exception handling http request", e); - internalServerErrorCounter.increment(); - return new StatusHolder(createStatus(e, Status.Code.INTERNAL), createHttpStatusFromProtoBufStatus(Status.Code.INTERNAL)); - } - } - - private HttpStatus createHttpStatusFromProtoBufStatus(Status.Code status) { - if (status == Status.Code.RESOURCE_EXHAUSTED) { - return HttpStatus.INSUFFICIENT_STORAGE; - } else if (status == Status.Code.INVALID_ARGUMENT) { - return HttpStatus.BAD_REQUEST; - } else { - return HttpStatus.INTERNAL_SERVER_ERROR; - } - } - - private com.google.rpc.Status createStatus(final Throwable e, final Status.Code code) { - com.google.rpc.Status.Builder builder = com.google.rpc.Status.newBuilder().setCode(code.value()); - if (e instanceof RequestTimeoutException) { - builder.setMessage(ARMERIA_REQUEST_TIMEOUT_MESSAGE); - } else { - builder.setMessage(e.getMessage() == null ? code.name() :e.getMessage()); - } - if (code == Status.Code.RESOURCE_EXHAUSTED) { - builder.addDetails(Any.pack(retryInfoCalculator.createRetryInfo())); - } - return builder.build(); - } - - private static class StatusHolder { - private final HttpStatus httpStatus; - private final com.google.rpc.Status status; - - public StatusHolder(com.google.rpc.Status status, HttpStatus httpStatus) { - this.httpStatus = httpStatus; - this.status = status; - } - - public HttpStatus getHttpStatus() { - return httpStatus; - } - - public com.google.rpc.Status getStatus() { - return status; - } - } - -} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java index db41b8d4bc..f7bf19ba73 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsGrpcServiceTest.java @@ -1,11 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics; diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfigFixture.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfigFixture.java deleted file mode 100644 index 8d8c82c3b0..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceConfigFixture.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.otelmetrics; - -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_PORT; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASIC_AUTH_PASSWORD; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASIC_AUTH_USERNAME; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.CONFIG_HTTP_PATH; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.RETRY_INFO; - -import java.util.Map; - -import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.plugins.codec.CompressionOption; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.common.HttpData; - -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationScope; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.metrics.v1.Gauge; -import io.opentelemetry.proto.metrics.v1.Metric; -import io.opentelemetry.proto.metrics.v1.NumberDataPoint; -import io.opentelemetry.proto.metrics.v1.ResourceMetrics; -import io.opentelemetry.proto.metrics.v1.ScopeMetrics; -import io.opentelemetry.proto.resource.v1.Resource; - -public class OTelMetricsSourceConfigFixture { - - public static OTelMetricsSourceConfig createDefaultConfig() { - return createDefaultConfigBuilder().build(); - } - - public static OTelMetricsSourceConfig.OTelMetricsSourceConfigBuilder createDefaultConfigBuilder() { - return OTelMetricsSourceConfig.builder() - .healthCheck(true) - .httpPath(CONFIG_HTTP_PATH) - .port(DEFAULT_PORT) - .enableUnframedRequests(false) - .ssl(false) - .requestTimeoutInMillis(DEFAULT_REQUEST_TIMEOUT_MS) - .maxConnectionCount(10) - .threadCount(5) - .retryInfo(RETRY_INFO) - .compression(CompressionOption.NONE); - } - - public static OTelMetricsSourceConfig.OTelMetricsSourceConfigBuilder createBuilderForConfigWithSsl() { - return createDefaultConfigBuilder() - .ssl(true) - .useAcmCertForSSL(false) - .sslKeyCertChainFile("data/certificate/test_cert.crt") - .sslKeyFile("data/certificate/test_decrypted_key.key"); - } - - public static OTelMetricsSourceConfig.OTelMetricsSourceConfigBuilder createBuilderForConfigWithAcmSsl() { - return createDefaultConfigBuilder() - .ssl(true) - .useAcmCertForSSL(true) - .awsRegion("us-east-1") - .acmCertificateArn("arn:aws:acm:us-east-1:account:certificate/1234-567-856456") - .sslKeyCertChainFile("data/certificate/test_cert.crt") - .sslKeyFile("data/certificate/test_decrypted_key.key"); - } - - public static OTelMetricsSourceConfig.OTelMetricsSourceConfigBuilder createConfigBuilderWithBasicAuth() { - Map httpBasicConfig = Map.of( - "username", BASIC_AUTH_USERNAME, - "password", BASIC_AUTH_PASSWORD - ); - return createDefaultConfigBuilder() - .authentication(new PluginModel("http_basic", httpBasicConfig)); - } - - public static ExportMetricsServiceRequest createMetricsServiceRequest() { - final Resource resource = Resource.newBuilder() - .addAttributes(KeyValue.newBuilder() - .setKey("service.name") - .setValue(AnyValue.newBuilder().setStringValue("service").build()) - ).build(); - - NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); - Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); - - Metric metric = Metric.newBuilder().setGauge(gauge).setUnit("seconds") - .setName("name") - .setDescription("description") - .build(); - - ScopeMetrics.Builder scopeMetric = ScopeMetrics.newBuilder() - .addMetrics(metric) - .setScope(InstrumentationScope.newBuilder() - .setName("scopeName") - .setVersion("scopeVersion") - .build()); - - ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() - .setResource(resource) - .addScopeMetrics(scopeMetric) - .build(); - - return ExportMetricsServiceRequest.newBuilder() - .addResourceMetrics(resourceMetrics) - .build(); - } - - public static HttpData createJsonHttpPayload() throws InvalidProtocolBufferException { - return HttpData.copyOf(JsonFormat.printer().print(createMetricsServiceRequest()).getBytes()); - } -} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceGrpcTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceGrpcTest.java deleted file mode 100644 index 0feb84914a..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceGrpcTest.java +++ /dev/null @@ -1,457 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.otelmetrics; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyCollection; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createBuilderForConfigWithAcmSsl; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createBuilderForConfigWithSsl; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createConfigBuilderWithBasicAuth; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createDefaultConfig; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createDefaultConfigBuilder; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createMetricsServiceRequest; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASE_64_ENCODED_BASIC_AUTH_CREDENTIALS; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASIC_AUTH_PASSWORD; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASIC_AUTH_USERNAME; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Base64; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; -import java.util.stream.Stream; - -import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; -import org.opensearch.dataprepper.metrics.MetricsTestUtil; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.buffer.SizeOverflowException; -import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.metric.Metric; -import org.opensearch.dataprepper.model.plugin.PluginFactory; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; -import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; -import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.model.Certificate; -import org.opensearch.dataprepper.plugins.source.otelmetrics.certificate.CertificateProviderFactory; - -import com.linecorp.armeria.client.Clients; -import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.server.Server; -import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.grpc.GrpcService; -import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; - -import io.grpc.BindableService; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.health.v1.HealthCheckRequest; -import io.grpc.health.v1.HealthCheckResponse; -import io.grpc.health.v1.HealthGrpc; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; -import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; - -@ExtendWith(MockitoExtension.class) -class OTelMetricsSourceGrpcTest { - private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21891/"; - private static final String TEST_PIPELINE_NAME = "test_pipeline"; - - @Mock - private ServerBuilder serverBuilder; - - @Mock - private Server server; - - @Mock - private GrpcServiceBuilder grpcServiceBuilder; - - @Mock - private GrpcService grpcService; - - @Mock - private CertificateProviderFactory certificateProviderFactory; - - @Mock - private CertificateProvider certificateProvider; - - @Mock - private Certificate certificate; - - @Mock - private CompletableFuture completableFuture; - - @Mock - private PluginFactory pluginFactory; - - @Mock - private GrpcBasicAuthenticationProvider authenticationProvider; - - @Mock - private ArmeriaHttpAuthenticationProvider httpAuthenticationProvider; - - @Mock - private BlockingBuffer> buffer; - - private PluginMetrics pluginMetrics; - private PipelineDescription pipelineDescription; - private OTelMetricsSource SOURCE; - - @BeforeEach - public void beforeEach() { - lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); - lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); - lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); - lenient().when(serverBuilder.build()).thenReturn(server); - lenient().when(serverBuilder.port(anyInt(), (SessionProtocol) any())).thenReturn(serverBuilder); - lenient().when(server.start()).thenReturn(completableFuture); - - lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); - - MetricsTestUtil.initMetrics(); - pluginMetrics = PluginMetrics.fromNames("otel_metrics", "pipeline"); - - lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authenticationProvider); - lenient().when(pluginFactory.loadPlugin(eq(ArmeriaHttpAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(httpAuthenticationProvider); - pipelineDescription = mock(PipelineDescription.class); - when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); - - setupSource(createDefaultConfig()); - } - - @AfterEach - public void afterEach() { - SOURCE.stop(); - } - - private void setupSource(OTelMetricsSourceConfig config) { - SOURCE = new OTelMetricsSource(config, pluginMetrics, pluginFactory, pipelineDescription); - } - - @Test - void testServerStartCertFileSuccess() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(server.stop()).thenReturn(completableFuture); - - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - - final OTelMetricsSource source = new OTelMetricsSource(createBuilderForConfigWithSsl().build(), pluginMetrics, pluginFactory, pipelineDescription); - source.start(buffer); - source.stop(); - - final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); - final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); - verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); - final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); - final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); - - assertThat(actualCertificate, is(certAsString)); - assertThat(actualPrivateKey, is(keyAsString)); - } - } - - @Test - void testServerStartACMCertSuccess() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final OTelMetricsSource source = new OTelMetricsSource(createBuilderForConfigWithAcmSsl().build(), pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - - final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); - final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); - verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); - final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); - final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); - - assertThat(actualCertificate, is(certAsString)); - assertThat(actualPrivateKey, is(keyAsString)); - } - } - - @Test - void startWithHealthConfiguredIncludesHealthCheckService() throws IOException { - SOURCE.start(buffer); - HealthGrpc.HealthBlockingStub healthClient = Clients.builder(GRPC_ENDPOINT).build(HealthGrpc.HealthBlockingStub.class); - HealthCheckResponse healthCheckResponse = healthClient.check(HealthCheckRequest.newBuilder().build()); - - assertThat(healthCheckResponse.getStatus().name(), equalTo("SERVING")); - } - - @Test - void startWithoutHealthConfiguredDoesNotIncludeHealthCheckService() throws IOException { - setupSource(createDefaultConfigBuilder().healthCheck(false).build()); - SOURCE.start(buffer); - HealthGrpc.HealthBlockingStub healthClient = Clients.builder(GRPC_ENDPOINT).build(HealthGrpc.HealthBlockingStub.class); - - StatusRuntimeException actualException = assertThrows( - StatusRuntimeException.class, - () -> healthClient.check(HealthCheckRequest.newBuilder().build()) - ); - - assertEquals(Status.Code.UNIMPLEMENTED, actualException.getStatus().getCode()); - } - - @Test - void testDoubleStart() { - SOURCE.start(buffer); - assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); - } - - @Test - void testRunAnotherSourceWithSamePort() { - SOURCE.start(buffer); - - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - - //Expect RuntimeException because when port is already in use, BindException is thrown which is not RuntimeException - assertThrows(RuntimeException.class, () -> source.start(buffer)); - } - - @Test - void testStartWithEmptyBuffer() { - assertThrows(IllegalStateException.class, () -> SOURCE.start(null)); - } - - @Test - void testStartWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(completableFuture.get()).thenThrow(new ExecutionException("", null)); - - // When/Then - assertThrows(RuntimeException.class, () -> source.start(buffer)); - } - } - - @Test - void testStartWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - final NullPointerException expCause = new NullPointerException(); - when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); - - // When/Then - final RuntimeException ex = assertThrows(RuntimeException.class, () -> source.start(buffer)); - assertEquals(expCause, ex); - } - } - - @Test - void testStopWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - when(server.stop()).thenReturn(completableFuture); - - // When/Then - when(completableFuture.get()).thenThrow(new ExecutionException("", null)); - assertThrows(RuntimeException.class, source::stop); - } - } - - @Test - void testStartWithInterruptedException() throws ExecutionException, InterruptedException { - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(completableFuture.get()).thenThrow(new InterruptedException()); - - // When/Then - assertThrows(RuntimeException.class, () -> source.start(buffer)); - assertTrue(Thread.interrupted()); - } - } - - @Test - void testStopWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - when(server.stop()).thenReturn(completableFuture); - final NullPointerException expCause = new NullPointerException(); - when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); - - // When/Then - final RuntimeException ex = assertThrows(RuntimeException.class, source::stop); - assertEquals(expCause, ex); - } - } - - @Test - void testStopWithInterruptedException() throws ExecutionException, InterruptedException { - // Prepare - final OTelMetricsSource source = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - when(server.stop()).thenReturn(completableFuture); - when(completableFuture.get()).thenThrow(new InterruptedException()); - - // When/Then - assertThrows(RuntimeException.class, source::stop); - assertTrue(Thread.interrupted()); - } - } - - @Test - void grpcRequestWritesToBufferWithSuccessfulResponse() throws Exception { - SOURCE.start(buffer); - - final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); - - final ExportMetricsServiceResponse exportResponse = client.export(createMetricsServiceRequest()); - assertThat(exportResponse, notNullValue()); - - final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); - verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); - - final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites, hasSize(1)); - } - - @Test - void grpcWithAuthRequestWritesToBufferWithSuccessfulResponse() throws Exception { - GrpcBasicAuthenticationProvider authProvider = new GrpcBasicAuthenticationProvider(new HttpBasicAuthenticationConfig(BASIC_AUTH_USERNAME, BASIC_AUTH_PASSWORD)); - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authProvider); - setupSource(createConfigBuilderWithBasicAuth().build()); - SOURCE.start(buffer); - final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .addHeader("Authorization", "Basic " + BASE_64_ENCODED_BASIC_AUTH_CREDENTIALS) - .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); - - final ExportMetricsServiceResponse exportResponse = client.export(createMetricsServiceRequest()); - - assertThat(exportResponse, notNullValue()); - final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); - verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); - final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites, hasSize(1)); - } - - @Test - void grpcWithAuthRequestWithDifferentBasicAuthCredentialsDoesNotWriteToBufferWith401Response() throws Exception { - GrpcBasicAuthenticationProvider authProvider = new GrpcBasicAuthenticationProvider(new HttpBasicAuthenticationConfig("username", "wrong password")); - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authProvider); - setupSource(createConfigBuilderWithBasicAuth().healthCheck(true).build()); - SOURCE.start(buffer); - final String givenCredentials = Base64.getEncoder().encodeToString(String.format("%s:%s", "wrong Username", "wrong Password").getBytes(StandardCharsets.UTF_8)); - final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .addHeader("Authorization", "Basic " + givenCredentials) - .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); - - StatusRuntimeException actualException = assertThrows( - StatusRuntimeException.class, - () -> client.export(createMetricsServiceRequest()) - ); - - assertEquals(Status.Code.UNAUTHENTICATED, actualException.getStatus().getCode()); - verify(buffer, never()).writeAll(any(), anyInt()); - } - - @ParameterizedTest - @ArgumentsSource(BufferExceptionToStatusArgumentsProvider.class) - void grpcRequestReturnsExpectedStatusForExceptionsFromBuffer( - final Class bufferExceptionClass, - final Status.Code expectedStatusCode) throws Exception { - SOURCE.start(buffer); - - final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); - - doThrow(bufferExceptionClass) - .when(buffer) - .writeAll(anyCollection(), anyInt()); - final io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest exportMetricsRequest = createMetricsServiceRequest(); - final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportMetricsRequest)); - - assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(expectedStatusCode)); - } - - static class BufferExceptionToStatusArgumentsProvider implements ArgumentsProvider { - @Override - public Stream provideArguments(final ExtensionContext context) { - return Stream.of( - arguments(TimeoutException.class, Status.Code.RESOURCE_EXHAUSTED), - arguments(SizeOverflowException.class, Status.Code.RESOURCE_EXHAUSTED), - arguments(Exception.class, Status.Code.INTERNAL), - arguments(RuntimeException.class, Status.Code.INTERNAL) - ); - } - } -} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceHttpTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceHttpTest.java deleted file mode 100644 index 866de6b05a..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceHttpTest.java +++ /dev/null @@ -1,431 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.otelmetrics; - -import static com.linecorp.armeria.common.HttpStatus.INSUFFICIENT_STORAGE; -import static com.linecorp.armeria.common.HttpStatus.INTERNAL_SERVER_ERROR; -import static com.linecorp.armeria.common.HttpStatus.REQUEST_ENTITY_TOO_LARGE; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyCollection; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createBuilderForConfigWithSsl; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createConfigBuilderWithBasicAuth; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createDefaultConfig; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createDefaultConfigBuilder; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createJsonHttpPayload; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createMetricsServiceRequest; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASIC_AUTH_PASSWORD; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.BASIC_AUTH_USERNAME; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OtelMetricsSourceConfigTestData.CONFIG_HTTP_PATH; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.zip.GZIPOutputStream; - -import io.opentelemetry.proto.metrics.v1.ResourceMetrics; -import io.opentelemetry.proto.metrics.v1.ScopeMetrics; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.verification.VerificationMode; -import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; -import org.opensearch.dataprepper.metrics.MetricsTestUtil; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.buffer.SizeOverflowException; -import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.metric.Metric; -import org.opensearch.dataprepper.model.plugin.PluginFactory; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.model.types.ByteCount; -import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; -import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; -import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; -import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.otel.codec.OTelMetricDecoder; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.ClientFactory; -import com.linecorp.armeria.client.WebClient; -import com.linecorp.armeria.common.AggregatedHttpResponse; -import com.linecorp.armeria.common.HttpData; -import com.linecorp.armeria.common.HttpHeaderNames; -import com.linecorp.armeria.common.HttpMethod; -import com.linecorp.armeria.common.HttpStatus; -import com.linecorp.armeria.common.MediaType; -import com.linecorp.armeria.common.RequestHeaders; -import com.linecorp.armeria.common.RequestHeadersBuilder; -import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.server.Server; -import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.grpc.GrpcService; -import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; - -import io.grpc.BindableService; -import io.micrometer.core.instrument.Measurement; -import io.micrometer.core.instrument.Statistic; -import io.netty.util.AsciiString; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.resource.v1.Resource; - - -@ExtendWith(MockitoExtension.class) -class OTelMetricsSourceHttpTest { - private static final String TEST_PIPELINE_NAME = "test_pipeline"; - - @Mock - private ServerBuilder serverBuilder; - - @Mock - private Server server; - - @Mock - private GrpcServiceBuilder grpcServiceBuilder; - - @Mock - private GrpcService grpcService; - - @Mock - private PluginFactory pluginFactory; - - @Mock - private GrpcBasicAuthenticationProvider authenticationProvider; - - @Mock - private BlockingBuffer> buffer; - - private PluginMetrics pluginMetrics; - private PipelineDescription pipelineDescription; - private OTelMetricsSource SOURCE; - private static final ExportMetricsServiceRequest METRICS_REQUEST = ExportMetricsServiceRequest.newBuilder() - .addResourceMetrics(ResourceMetrics.newBuilder().build()).build(); - - @BeforeEach - public void beforeEach() { - lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); - lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); - lenient().when(serverBuilder.build()).thenReturn(server); - - lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); - - MetricsTestUtil.initMetrics(); - pluginMetrics = PluginMetrics.fromNames("otel_metrics", "pipeline"); - - lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authenticationProvider); - pipelineDescription = mock(PipelineDescription.class); - when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); - } - - @AfterEach - public void afterEach() { - SOURCE.stop(); - } - - private void configureSource() { - configureSource(createDefaultConfig()); - } - - private void configureSource(OTelMetricsSourceConfig config) { - SOURCE = new OTelMetricsSource(config, pluginMetrics, pluginFactory, pipelineDescription); - assertInstanceOf(OTelMetricDecoder.class, SOURCE.getDecoder()); - } - - private RequestHeadersBuilder getDefaultRequestHeadersBuilder() { - return RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21891") - .method(HttpMethod.POST) - .path(CONFIG_HTTP_PATH) - .contentType(MediaType.JSON_UTF_8); - } - - @ParameterizedTest - @MethodSource("getPathParams") - void httpRequest_writesToBuffer_returnsSuccessfulResponse(String givenPath, String resolvedRequestPath) throws Exception { - OTelMetricsSourceConfig config = createDefaultConfigBuilder().httpPath(givenPath).build(); - configureSource(config); - SOURCE.start(buffer); - ExportMetricsServiceRequest request = createMetricsServiceRequest(); - - WebClient.of().execute( - getDefaultRequestHeadersBuilder().path(resolvedRequestPath).scheme(SessionProtocol.HTTP).build(), - HttpData.copyOf(JsonFormat.printer().print(request).getBytes()) - ) - .aggregate() - .whenComplete((response, throwable) -> assertThat(response.status(), is(HttpStatus.OK))) - .join(); - - verify(buffer).writeAll(any(), anyInt()); - } - - @Test - void httpRequest_payloadIsProtobuf_returnsSuccessfulResponse() throws Exception { - configureSource(); - SOURCE.start(buffer); - ExportMetricsServiceRequest request = createMetricsServiceRequest(); - - WebClient.of().execute( - getDefaultRequestHeadersBuilder().contentType(MediaType.PROTOBUF).scheme(SessionProtocol.HTTP).build(), - HttpData.copyOf(request.toByteArray()) - ) - .aggregate() - .whenComplete((response, throwable) -> assertThat(response.status(), is(HttpStatus.OK))) - .join(); - - verify(buffer).writeAll(any(), anyInt()); - } - - @Test - void httpsRequest_requestIsProcessed_writesToBufferAndReturnsSuccessfulResponse() throws Exception { - configureSource(createBuilderForConfigWithSsl().build()); - SOURCE.start(buffer); - ExportMetricsServiceRequest request = createExportMetricsRequest(); - - WebClient.builder() - .factory(ClientFactory.insecure()). - build().execute( - getDefaultRequestHeadersBuilder().scheme(SessionProtocol.HTTPS).build(), - HttpData.copyOf(JsonFormat.printer().print(request).getBytes()) - ) - .aggregate() - .whenComplete((response, throwable) -> assertThat(response.status(), is(HttpStatus.OK))) - .join(); - - verify(buffer).writeAll(any(), anyInt()); - } - - - @Test - void httpRequest_oneConnectionIsEstablished_metricsReflectCorrectConnectionCount() throws InvalidProtocolBufferException { - configureSource(); - SOURCE.start(buffer); - - WebClient.of().execute(getDefaultRequestHeadersBuilder().build(), createJsonHttpPayload()) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - - List serverConnectionsMeasurements = MetricsTestUtil.getMeasurementList("pipeline.otel_metrics.serverConnections"); - Measurement serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); - assertEquals(1.0, serverConnectionsMeasurement.getValue()); - - SOURCE.stop(); - } - - @Test - void httpRequest_payloadIsCompressed_returns200() throws IOException { - configureSource( createDefaultConfigBuilder().compression(CompressionOption.GZIP).build()); - SOURCE.start(buffer); - - WebClient.of().execute(getDefaultRequestHeadersBuilder() - .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") - .build(), - createGZipCompressedPayload(JsonFormat.printer().print(createMetricsServiceRequest()))) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @ParameterizedTest - @MethodSource("getBasicAuthTestData") - void httpRequest_withBasicAuth_returnsAppropriateResponse(String givenUsername, String givenPassword, HttpStatus expectedStatus, VerificationMode expectedBufferWrites) throws Exception { - final HttpBasicAuthenticationConfig basicAuthConfig = new HttpBasicAuthenticationConfig(BASIC_AUTH_USERNAME, BASIC_AUTH_PASSWORD); - final HttpBasicArmeriaHttpAuthenticationProvider authProvider = new HttpBasicArmeriaHttpAuthenticationProvider(basicAuthConfig); - when(pluginFactory.loadPlugin(eq(ArmeriaHttpAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authProvider); - configureSource(createConfigBuilderWithBasicAuth().build()); - SOURCE.start(buffer); - - final String encodedCredentials = Base64.getEncoder().encodeToString(String.format("%s:%s", givenUsername, givenPassword).getBytes(StandardCharsets.UTF_8)); - WebClient.of().execute(getDefaultRequestHeadersBuilder() - .add(HttpHeaderNames.AUTHORIZATION, "Basic " + encodedCredentials) - .build(), - HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, expectedStatus, throwable)) - .join(); - - verify(buffer, expectedBufferWrites).writeAll(any(), anyInt()); - } - - private static Stream getBasicAuthTestData() { - return Stream.of( - Arguments.of(BASIC_AUTH_USERNAME, BASIC_AUTH_PASSWORD, HttpStatus.OK, times(1)), - Arguments.of(BASIC_AUTH_USERNAME, "wrong password", HttpStatus.UNAUTHORIZED, times(0)) - ); - } - - @ParameterizedTest - @MethodSource("getHealthCheckParams") - void healthCheckRequest_requestIsProcessed_returnsStatusCodeAccordingToConfig(boolean givenHealthCheckConfig, HttpStatus expectedStatus) throws IOException { - configureSource(createDefaultConfigBuilder().healthCheck(givenHealthCheckConfig).build()); - SOURCE.start(buffer); - - WebClient.of().execute(getDefaultRequestHeadersBuilder() - .path("/health") - .method(HttpMethod.GET) - .build(), - HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, expectedStatus, throwable)) - .join(); - } - - private static Stream getHealthCheckParams() { - return Stream.of( - Arguments.of(true, HttpStatus.OK), - Arguments.of(false, HttpStatus.NOT_FOUND) - ); - } - - @Test - void testStartWithEmptyBuffer() { - configureSource(); - assertThrows(IllegalStateException.class, () -> SOURCE.start(null)); - } - - @ParameterizedTest - @ArgumentsSource(BufferExceptionToStatusArgumentsProvider.class) - void httpRequest_writingToBufferThrowsAnException_correctHttpStatusIsReturned( - final Class bufferExceptionClass, - final HttpStatus expectedStatus) throws Exception { - configureSource(); - SOURCE.start(buffer); - doThrow(bufferExceptionClass) - .when(buffer) - .writeAll(anyCollection(), anyInt()); - - WebClient.of() - .execute(getDefaultRequestHeadersBuilder().build(), createJsonHttpPayload()) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, expectedStatus, throwable)) - .join(); - - SOURCE.stop(); - } - - @Test - void httpRequest_requestBodyIsTooLarge_returns413() throws InvalidProtocolBufferException { - configureSource(createDefaultConfigBuilder().maxRequestLength(ByteCount.ofBytes(4)).build()); - SOURCE.start(buffer); - - WebClient.of() - .execute(getDefaultRequestHeadersBuilder().build(), createJsonHttpPayload()) - .aggregate() - .whenComplete((response, throwable) -> { - assertSecureResponseWithStatusCode(response, REQUEST_ENTITY_TOO_LARGE, throwable); - }) - .join(); - } - - static class BufferExceptionToStatusArgumentsProvider implements ArgumentsProvider { - @Override - public Stream provideArguments(final ExtensionContext context) { - return Stream.of( - arguments(TimeoutException.class, INSUFFICIENT_STORAGE), - arguments(SizeOverflowException.class, INSUFFICIENT_STORAGE), - arguments(Exception.class, INTERNAL_SERVER_ERROR), - arguments(RuntimeException.class, INTERNAL_SERVER_ERROR) - ); - } - } - - private ExportMetricsServiceRequest createExportMetricsRequest() { - final Resource resource = Resource.newBuilder() - .addAttributes(KeyValue.newBuilder() - .setKey("service.name") - .setValue(AnyValue.newBuilder().setStringValue("service").build()) - ).build(); - - final ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() - .addScopeMetrics(ScopeMetrics.newBuilder() - .addMetrics(io.opentelemetry.proto.metrics.v1.Metric.newBuilder().build()) - .build()) - .setResource(resource) - .build(); - - return ExportMetricsServiceRequest.newBuilder() - .addResourceMetrics(resourceMetrics) - .build(); - } - - private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, - final HttpStatus expectedStatus, - final Throwable throwable) { - assertThat("Http Status", response.status(), equalTo(expectedStatus)); - assertThat("Http Response Throwable", throwable, is(nullValue())); - - final List headerKeys = response.headers() - .stream() - .map(Map.Entry::getKey) - .map(AsciiString::toString) - .map(String::toLowerCase) - .collect(Collectors.toList()); - assertThat("Response Header Keys", headerKeys, not(hasItem("server"))); - } - - private byte[] createGZipCompressedPayload(final String payload) throws IOException { - // Create a GZip compressed request body - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { - gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); - } - return byteStream.toByteArray(); - } - - private static Stream getPathParams() { - return Stream.of( - Arguments.of(CONFIG_HTTP_PATH, CONFIG_HTTP_PATH), - Arguments.of("/${pipelineName}/v1/metrics", "/test_pipeline/v1/metrics") - ); - } -} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java new file mode 100644 index 0000000000..8e41255b0e --- /dev/null +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java @@ -0,0 +1,1228 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.otelmetrics; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.ClosedSessionException; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; +import io.grpc.BindableService; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.instrument.Measurement; +import io.micrometer.core.instrument.Statistic; +import io.netty.util.AsciiString; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.common.v1.InstrumentationScope; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; + +import io.opentelemetry.proto.metrics.v1.Gauge; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; +import io.opentelemetry.proto.resource.v1.Resource; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; +import org.opensearch.dataprepper.metrics.MetricNames; +import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.metric.Metric; +import org.opensearch.dataprepper.model.types.ByteCount; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.server.HealthGrpcService; +import org.opensearch.dataprepper.plugins.otel.codec.OTelMetricDecoder; +import org.opensearch.dataprepper.plugins.source.otelmetrics.certificate.CertificateProviderFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPOutputStream; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.SSL; + +@ExtendWith(MockitoExtension.class) +class OTelMetricsSourceTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21891/"; + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final String USERNAME = "test_user"; + private static final String PASSWORD = "test_password"; + private static final String TEST_PATH = "${pipelineName}/v1/metrics"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ExportMetricsServiceRequest METRICS_REQUEST = ExportMetricsServiceRequest.newBuilder() + .addResourceMetrics(ResourceMetrics.newBuilder().build()).build(); + + @Mock + private ServerBuilder serverBuilder; + + @Mock + private Server server; + + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + + @Mock + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; + + @Mock + private Certificate certificate; + + @Mock + private CompletableFuture completableFuture; + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelMetricsSourceConfig oTelMetricsSourceConfig; + + @Mock + private BlockingBuffer> buffer; + + @Mock + private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; + + private PluginSetting pluginSetting; + private PluginSetting testPluginSetting; + private PluginMetrics pluginMetrics; + private PipelineDescription pipelineDescription; + private OTelMetricsSource SOURCE; + + @BeforeEach + public void beforeEach() { + lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.build()).thenReturn(server); + lenient().when(server.start()).thenReturn(completableFuture); + + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + MetricsTestUtil.initMetrics(); + pluginMetrics = PluginMetrics.fromNames("otel_metrics", "pipeline"); + + when(oTelMetricsSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelMetricsSourceConfig.isSsl()).thenReturn(false); + when(oTelMetricsSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelMetricsSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelMetricsSourceConfig.getThreadCount()).thenReturn(5); + when(oTelMetricsSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + assertTrue(SOURCE.getDecoder() instanceof OTelMetricDecoder); + } + + @AfterEach + public void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + assertTrue(SOURCE.getDecoder() instanceof OTelMetricDecoder); + } + + @Test + void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpsFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put("request_timeout", 5); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + pluginSetting = new PluginSetting("otel_metrics", settingsMap); + pluginSetting.setPipelineName("pipeline"); + + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelMetricsSourceConfig.class); + SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + SOURCE.start(buffer); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullBytesWithNonUnframedRequests() { + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(METRICS_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithUnframedRequests() throws InvalidProtocolBufferException { + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpCompressionWithUnframedRequests() throws IOException { + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getCompression()).thenReturn(CompressionOption.GZIP); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") + .build(), + createGZipCompressedPayload(JsonFormat.printer().print(METRICS_REQUEST))) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolBufferException { + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); + SOURCE.start(buffer); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path(transformedPath) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndAuthHeader_with_successful_response() throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelMetricsSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String encodeToString = Base64.getEncoder() + .encodeToString(String.format("%s:%s", USERNAME, PASSWORD).getBytes(StandardCharsets.UTF_8)); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + + WebClient.of().prepare() + .post("http://127.0.0.1:21891" + transformedPath) + .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) + .header("Authorization", "Basic " + encodeToString) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndAuthHeader_with_unsuccessful_response() throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelMetricsSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + + WebClient.of().prepare() + .post("http://127.0.0.1:21891" + transformedPath) + .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) + .join(); + } + + @Test + void testHttpRequestWithInvalidCredentials_with_unsuccessful_response() throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelMetricsSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String invalidUsername = "wrong_user"; + final String invalidPassword = "wrong_password"; + final String invalidCredentials = Base64.getEncoder() + .encodeToString(String.format("%s:%s", invalidUsername, invalidPassword).getBytes(StandardCharsets.UTF_8)); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; + + WebClient.of().prepare() + .post("http://127.0.0.1:21891" + transformedPath) + .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) + .header("Authorization", "Basic " + invalidCredentials) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) + .join(); + } + + @Test + void testGrpcRequestWithInvalidCredentials_with_unsuccessful_response() throws Exception { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelMetricsSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String invalidUsername = "wrong_user"; + final String invalidPassword = "wrong_password"; + final String invalidCredentials = Base64.getEncoder() + .encodeToString(String.format("%s:%s", invalidUsername, invalidPassword).getBytes(StandardCharsets.UTF_8)); + + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .addHeader("Authorization", "Basic " + invalidCredentials) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.UNAUTHENTICATED)); + } + + @Test + void testHttpWithoutSslFailsWhenSslIsEnabled() throws InvalidProtocolBufferException { + when(oTelMetricsSourceConfig.isSsl()).thenReturn(true); + when(oTelMetricsSourceConfig.getSslKeyCertChainFile()).thenReturn("data/certificate/test_cert.crt"); + when(oTelMetricsSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient client = WebClient.builder("http://127.0.0.1:21891") + .build(); + + CompletionException exception = assertThrows(CompletionException.class, () -> client.execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportMetricsRequest()).getBytes())) + .aggregate() + .join()); + + assertThat(exception.getCause(), instanceOf(ClosedSessionException.class)); + } + + @Test + void testGrpcFailsIfSslIsEnabledAndNoTls() { + when(oTelMetricsSourceConfig.isSsl()).thenReturn(true); + when(oTelMetricsSourceConfig.getSslKeyCertChainFile()).thenReturn("data/certificate/test_cert.crt"); + when(oTelMetricsSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); + configureObjectUnderTest(); + SOURCE.start(buffer); + + MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + + StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.UNKNOWN)); + } + + + @Test + void testServerStartCertFileSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate, is(certAsString)); + assertThat(actualPrivateKey, is(keyAsString)); + } + } + + @Test + void testServerStartACMCertSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate, is(certAsString)); + assertThat(actualPrivateKey, is(keyAsString)); + } + } + + @Test + void start_with_Health_configured_includes_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + @Test + void start_with_Health_configured_includes_HealthCheck_service_Unauthenticated_disabled() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + @Nested + class UnauthTests { + final Map settingsMap = new HashMap<>(); + + @BeforeEach + void setup() { + when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + + settingsMap.clear(); + settingsMap.put(SSL, false); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + settingsMap.put("proto_reflection_service", "true"); + settingsMap.put("authentication", new PluginModel("http_basic", + Map.of( + "username", "test", + "password", "test2" + ))); + } + + void createObjectUnderTest() { + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + assertTrue(SOURCE.getDecoder() instanceof OTelMetricDecoder); + } + + @Test + void testHealthCheckUnauthNotAllowed() { + settingsMap.put("unauthenticated_health_check", "false"); + createObjectUnderTest(); + + SOURCE.start(buffer); + + // When + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.GET) + .path("/health") + .build()) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) + .join(); + } + + @Test + void testHealthCheckUnauthAllowed() { + settingsMap.put("unauthenticated_health_check", "true"); + createObjectUnderTest(); + + SOURCE.start(buffer); + + // When + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.GET) + .path("/health") + .build()) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + } + + @Test + void start_with_Health_configured_unframed_requests_includes_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder).service(eq("/health"), isA(HealthCheckService.class)); + } + + @Test + void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + @Test + void start_without_Health_configured_unframed_requests_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + @Test + void testDoubleStart() { + // starting server + SOURCE.start(buffer); + // double start server + assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); + } + + @Test + void testRunAnotherSourceWithSamePort() { + // starting server + SOURCE.start(buffer); + + testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); + testPluginSetting.setPipelineName("pipeline"); + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + //Expect RuntimeException because when port is already in use, BindException is thrown which is not RuntimeException + assertThrows(RuntimeException.class, () -> source.start(buffer)); + } + + @Test + void testStartWithEmptyBuffer() { + testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); + testPluginSetting.setPipelineName("pipeline"); + oTelMetricsSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + assertThrows(IllegalStateException.class, () -> source.start(null)); + } + + @Test + void testStartWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(completableFuture.get()).thenThrow(new ExecutionException("", null)); + + // When/Then + assertThrows(RuntimeException.class, () -> source.start(buffer)); + } + } + + @Test + void testStartWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + final NullPointerException expCause = new NullPointerException(); + when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); + + // When/Then + final RuntimeException ex = assertThrows(RuntimeException.class, () -> source.start(buffer)); + assertEquals(expCause, ex); + } + } + + @Test + void testOptionalHttpAuthServiceNotInPlace() { + when(server.stop()).thenReturn(completableFuture); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + SOURCE.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder, never()).decorator(isA(Function.class)); + } + + @Test + void testOptionalHttpAuthServiceInPlace() { + when(server.stop()).thenReturn(completableFuture); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + SOURCE.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + } + + @Test + void testStopWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + + // When/Then + when(completableFuture.get()).thenThrow(new ExecutionException("", null)); + assertThrows(RuntimeException.class, source::stop); + } + } + + @Test + void testStartWithInterruptedException() throws ExecutionException, InterruptedException { + // Prepare + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(completableFuture.get()).thenThrow(new InterruptedException()); + + // When/Then + assertThrows(RuntimeException.class, () -> source.start(buffer)); + assertTrue(Thread.interrupted()); + } + } + + @Test + void testStopWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + final NullPointerException expCause = new NullPointerException(); + when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); + + // When/Then + final RuntimeException ex = assertThrows(RuntimeException.class, source::stop); + assertEquals(expCause, ex); + } + } + + @Test + void testStopWithInterruptedException() throws ExecutionException, InterruptedException { + // Prepare + final OTelMetricsSource source = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + when(completableFuture.get()).thenThrow(new InterruptedException()); + + // When/Then + assertThrows(RuntimeException.class, source::stop); + assertTrue(Thread.interrupted()); + } + } + + @Test + void gRPC_request_writes_to_buffer_with_successful_response() throws Exception { + SOURCE.start(buffer); + + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final ExportMetricsServiceResponse exportResponse = client.export(createExportMetricsRequest()); + assertThat(exportResponse, notNullValue()); + + final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); + verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); + + final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); + assertThat(actualBufferWrites, notNullValue()); + assertThat(actualBufferWrites.size(), equalTo(1)); + } + + @Test + void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws Exception { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String encodeToString = Base64.getEncoder() + .encodeToString(String.format("%s:%s", USERNAME, PASSWORD).getBytes(StandardCharsets.UTF_8)); + + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .addHeader("Authorization", "Basic " + encodeToString) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + final ExportMetricsServiceResponse exportResponse = client.export(createExportMetricsRequest()); + assertThat(exportResponse, notNullValue()); + + final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); + verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); + + final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); + assertThat(actualBufferWrites, notNullValue()); + assertThat(actualBufferWrites.size(), equalTo(1)); + } + + @Test + void gRPC_request_with_custom_path_throws_when_written_to_default_path() { + when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); + } + + @ParameterizedTest + @ArgumentsSource(BufferExceptionToStatusArgumentsProvider.class) + void gRPC_request_returns_expected_status_for_exceptions_from_buffer( + final Class bufferExceptionClass, + final Status.Code expectedStatusCode) throws Exception { + SOURCE.start(buffer); + + final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); + + doThrow(bufferExceptionClass) + .when(buffer) + .writeAll(any(Collection.class), anyInt()); + final ExportMetricsServiceRequest exportMetricsRequest = createExportMetricsRequest(); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportMetricsRequest)); + + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(expectedStatusCode)); + } + + @Test + void request_that_exceeds_maxRequestLength_returns_413() throws InvalidProtocolBufferException { + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelMetricsSourceConfig.getMaxRequestLength()).thenReturn(ByteCount.ofBytes(4)); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) + .join(); + } + + @Test + void testServerConnectionsMetric() throws InvalidProtocolBufferException { + // Prepare + when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); + SOURCE.start(buffer); + + final String metricNamePrefix = new StringJoiner(MetricNames.DELIMITER) + .add("pipeline").add("otel_metrics").toString(); + List serverConnectionsMeasurements = MetricsTestUtil.getMeasurementList( + new StringJoiner(MetricNames.DELIMITER).add(metricNamePrefix) + .add(OTelMetricsSource.SERVER_CONNECTIONS).toString()); + + // Verify connections metric value is 0 + Measurement serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); + assertEquals(0, serverConnectionsMeasurement.getValue()); + + final RequestHeaders testRequestHeaders = RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21891") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.metrics.v1.MetricsService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(); + final HttpData testHttpData = HttpData.copyOf(JsonFormat.printer().print(METRICS_REQUEST).getBytes()); + + // Send request + WebClient.of().execute(testRequestHeaders, testHttpData) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + + // Verify connections metric value is 1 + serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); + assertEquals(1.0, serverConnectionsMeasurement.getValue()); + } + + static class BufferExceptionToStatusArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + arguments(TimeoutException.class, Status.Code.RESOURCE_EXHAUSTED), + arguments(RuntimeException.class, Status.Code.INTERNAL) + ); + } + } + + private ExportMetricsServiceRequest createExportMetricsRequest() { + final Resource resource = Resource.newBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey("service.name") + .setValue(AnyValue.newBuilder().setStringValue("service").build()) + ).build(); + NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); + Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); + + io.opentelemetry.proto.metrics.v1.Metric.Builder metric = io.opentelemetry.proto.metrics.v1.Metric.newBuilder() + .setGauge(gauge) + .setUnit("seconds") + .setName("name") + .setDescription("description"); + ScopeMetrics scopeMetrics = ScopeMetrics.newBuilder() + .addMetrics(metric) + .setScope(InstrumentationScope.newBuilder() + .setName("ilname") + .setVersion("ilversion") + .build()) + .build(); + + + final ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() + .setResource(resource) + .addScopeMetrics(scopeMetrics) + .build(); + + return ExportMetricsServiceRequest.newBuilder() + .addResourceMetrics(resourceMetrics).build(); + } + + private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, + final HttpStatus expectedStatus, + final Throwable throwable) { + assertThat("Http Status", response.status(), equalTo(expectedStatus)); + assertThat("Http Response Throwable", throwable, is(nullValue())); + + final List headerKeys = response.headers() + .stream() + .map(Map.Entry::getKey) + .map(AsciiString::toString) + .collect(Collectors.toList()); + assertThat("Response Header Keys", headerKeys, not(contains("server"))); + } + + private byte[] createGZipCompressedPayload(final String payload) throws IOException { + // Create a GZip compressed request body + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { + gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); + } + return byteStream.toByteArray(); + } +} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java similarity index 81% rename from data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java rename to data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java index eaf7e4747a..59627b70ec 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceRetryInfoTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSource_RetryInfoTest.java @@ -1,11 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics; @@ -19,7 +14,8 @@ import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfigFixture.createDefaultConfig; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.otelmetrics.OTelMetricsSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; import java.time.Duration; @@ -40,6 +36,7 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; import com.google.protobuf.InvalidProtocolBufferException; import com.google.rpc.RetryInfo; @@ -56,11 +53,13 @@ import io.opentelemetry.proto.metrics.v1.ResourceMetrics; import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; @ExtendWith(MockitoExtension.class) -class OTelMetricsSourceRetryInfoTest { +class OTelMetricsSource_RetryInfoTest { private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21891/"; private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); @Mock private PluginFactory pluginFactory; @@ -68,6 +67,9 @@ class OTelMetricsSourceRetryInfoTest { @Mock private GrpcBasicAuthenticationProvider authenticationProvider; + @Mock(lenient = true) + private OTelMetricsSourceConfig oTelMetricsSourceConfig; + @Mock private Buffer> buffer; @@ -75,15 +77,21 @@ class OTelMetricsSourceRetryInfoTest { @BeforeEach void beforeEach() throws Exception { - PluginMetrics pluginMetrics = PluginMetrics.fromNames("otel_metrics", "pipeline"); - PipelineDescription pipelineDescription = mock(PipelineDescription.class); - when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); - lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authenticationProvider); - SOURCE = new OTelMetricsSource(createDefaultConfig(), pluginMetrics, pluginFactory, pipelineDescription); + when(oTelMetricsSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelMetricsSourceConfig.isSsl()).thenReturn(false); + when(oTelMetricsSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelMetricsSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelMetricsSourceConfig.getThreadCount()).thenReturn(5); + when(oTelMetricsSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelMetricsSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + + configureObjectUnderTest(); SOURCE.start(buffer); } @@ -92,6 +100,14 @@ void afterEach() { SOURCE.stop(); } + private void configureObjectUnderTest() { + PluginMetrics pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); + + PipelineDescription pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelMetricsSource(oTelMetricsSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + @Test public void gRPC_failed_request_returns_minimal_delay_in_status() throws Exception { final MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTestData.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTestData.java deleted file mode 100644 index f9cdff0b4a..0000000000 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTestData.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.plugins.source.otelmetrics; - -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Base64; - -import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; - -public class OtelMetricsSourceConfigTestData { - public static final String CONFIG_HTTP_PATH = "/metrics/v1"; - public static final String BASIC_AUTH_USERNAME = "test"; - public static final String BASIC_AUTH_PASSWORD = "password"; - - public static final String BASE_64_ENCODED_BASIC_AUTH_CREDENTIALS = Base64.getEncoder() - .encodeToString(String.format("%s:%s", BASIC_AUTH_USERNAME, BASIC_AUTH_PASSWORD).getBytes(StandardCharsets.UTF_8)); - - public static final RetryInfoConfig RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); -} diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java index 00fdc92650..2f5ee30d00 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OtelMetricsSourceConfigTests.java @@ -1,11 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics; @@ -66,11 +61,11 @@ void testDefault() { assertEquals(DEFAULT_REQUEST_TIMEOUT_MS, otelMetricsSourceConfig.getRequestTimeoutInMillis()); assertEquals(DEFAULT_PORT, otelMetricsSourceConfig.getPort()); assertEquals(DEFAULT_THREAD_COUNT, otelMetricsSourceConfig.getThreadCount()); - assertEquals(DEFAULT_MAX_CONNECTION_COUNT, otelMetricsSourceConfig.getMaxConnectionCount()); + assertEquals(OTelMetricsSourceConfig.DEFAULT_MAX_CONNECTION_COUNT, otelMetricsSourceConfig.getMaxConnectionCount()); assertEquals(CompressionOption.NONE, otelMetricsSourceConfig.getCompression()); - assertFalse(otelMetricsSourceConfig.isHealthCheck()); + assertFalse(otelMetricsSourceConfig.hasHealthCheck()); assertFalse(otelMetricsSourceConfig.enableHttpHealthCheck()); - assertFalse(otelMetricsSourceConfig.isProtoReflectionService()); + assertFalse(otelMetricsSourceConfig.hasProtoReflectionService()); assertFalse(otelMetricsSourceConfig.isSslCertAndKeyFileInS3()); assertTrue(otelMetricsSourceConfig.isSsl()); assertNull(otelMetricsSourceConfig.getSslKeyCertChainFile()); @@ -104,9 +99,9 @@ void testHttpHealthCheckWithUnframedRequestEnabled() { final OTelMetricsSourceConfig otelMetricsSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelMetricsSourceConfig.class); // When/Then - assertTrue(otelMetricsSourceConfig.isHealthCheck()); - assertTrue(otelMetricsSourceConfig.isEnableUnframedRequests()); - assertTrue(otelMetricsSourceConfig.isProtoReflectionService()); + assertTrue(otelMetricsSourceConfig.hasHealthCheck()); + assertTrue(otelMetricsSourceConfig.enableUnframedRequests()); + assertTrue(otelMetricsSourceConfig.hasProtoReflectionService()); assertTrue(otelMetricsSourceConfig.enableHttpHealthCheck()); } @@ -122,9 +117,9 @@ void testHttpHealthCheckWithUnframedRequestDisabled() { final OTelMetricsSourceConfig otelMetricsSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelMetricsSourceConfig.class); // When/Then - assertTrue(otelMetricsSourceConfig.isHealthCheck()); - assertFalse(otelMetricsSourceConfig.isEnableUnframedRequests()); - assertTrue(otelMetricsSourceConfig.isProtoReflectionService()); + assertTrue(otelMetricsSourceConfig.hasHealthCheck()); + assertFalse(otelMetricsSourceConfig.enableUnframedRequests()); + assertTrue(otelMetricsSourceConfig.hasProtoReflectionService()); assertFalse(otelMetricsSourceConfig.enableHttpHealthCheck()); } @@ -153,8 +148,8 @@ void testValidConfigWithoutS3CertAndKey() { assertEquals(TEST_PORT, otelMetricsSourceConfig.getPort()); assertEquals(TEST_THREAD_COUNT, otelMetricsSourceConfig.getThreadCount()); assertEquals(TEST_MAX_CONNECTION_COUNT, otelMetricsSourceConfig.getMaxConnectionCount()); - assertTrue(otelMetricsSourceConfig.isHealthCheck()); - assertTrue(otelMetricsSourceConfig.isProtoReflectionService()); + assertTrue(otelMetricsSourceConfig.hasHealthCheck()); + assertTrue(otelMetricsSourceConfig.hasProtoReflectionService()); assertFalse(otelMetricsSourceConfig.enableHttpHealthCheck()); assertTrue(otelMetricsSourceConfig.isSsl()); assertFalse(otelMetricsSourceConfig.isSslCertAndKeyFileInS3()); @@ -188,9 +183,9 @@ void testValidConfigWithS3CertAndKey() { assertEquals(TEST_PORT, otelMetricsSourceConfig.getPort()); assertEquals(TEST_THREAD_COUNT, otelMetricsSourceConfig.getThreadCount()); assertEquals(TEST_MAX_CONNECTION_COUNT, otelMetricsSourceConfig.getMaxConnectionCount()); - assertFalse(otelMetricsSourceConfig.isHealthCheck()); + assertFalse(otelMetricsSourceConfig.hasHealthCheck()); assertFalse(otelMetricsSourceConfig.enableHttpHealthCheck()); - assertFalse(otelMetricsSourceConfig.isProtoReflectionService()); + assertFalse(otelMetricsSourceConfig.hasProtoReflectionService()); assertTrue(otelMetricsSourceConfig.isSsl()); assertTrue(otelMetricsSourceConfig.isSslCertAndKeyFileInS3()); assertEquals(TEST_KEY_CERT_S3, otelMetricsSourceConfig.getSslKeyCertChainFile()); @@ -327,9 +322,9 @@ void testRetryInfoConfig() { DEFAULT_THREAD_COUNT, DEFAULT_MAX_CONNECTION_COUNT); - final OTelMetricsSourceConfig otelMetricsSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelMetricsSourceConfig.class); + final OTelMetricsSourceConfig otelTraceSourceConfig = OBJECT_MAPPER.convertValue(customPathPluginSetting.getSettings(), OTelMetricsSourceConfig.class); - RetryInfoConfig retryInfo = otelMetricsSourceConfig.getRetryInfo(); + RetryInfoConfig retryInfo = otelTraceSourceConfig.getRetryInfo(); assertThat(retryInfo.getMaxDelay(), equalTo(Duration.ofMillis(100))); assertThat(retryInfo.getMinDelay(), equalTo(Duration.ofMillis(50))); } @@ -339,7 +334,7 @@ private PluginSetting completePluginSettingForOtelMetricsSource(final int reques final String path, final boolean healthCheck, final boolean protoReflectionService, - final boolean isEnableUnframedRequests, + final boolean enableUnframedRequests, final boolean isSSL, final String sslKeyCertChainFile, final String sslKeyFile, @@ -351,7 +346,7 @@ private PluginSetting completePluginSettingForOtelMetricsSource(final int reques settings.put(OTelMetricsSourceConfig.PATH, path); settings.put(OTelMetricsSourceConfig.HEALTH_CHECK_SERVICE, healthCheck); settings.put(OTelMetricsSourceConfig.PROTO_REFLECTION_SERVICE, protoReflectionService); - settings.put(OTelMetricsSourceConfig.ENABLE_UNFRAMED_REQUESTS, isEnableUnframedRequests); + settings.put(OTelMetricsSourceConfig.ENABLE_UNFRAMED_REQUESTS, enableUnframedRequests); settings.put(OTelMetricsSourceConfig.SSL, isSSL); settings.put(OTelMetricsSourceConfig.SSL_KEY_CERT_FILE, sslKeyCertChainFile); settings.put(OTelMetricsSourceConfig.SSL_KEY_FILE, sslKeyFile); diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactoryTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactoryTest.java index e139fde417..9e66abe3eb 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactoryTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/certificate/CertificateProviderFactoryTest.java @@ -1,11 +1,6 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * */ package org.opensearch.dataprepper.plugins.source.otelmetrics.certificate; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 5d7846199d..adac5a2940 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -63,6 +63,7 @@ import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.types.ByteCount; import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; @@ -407,6 +408,24 @@ void testGrpcRequestWithoutAuthentication_with_unsuccessful_response() throws Ex assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.UNAUTHENTICATED)); } + + @Test + void testGrpcFailsIfSslIsEnabledAndNoTls() { + when(oTelTraceSourceConfig.isSsl()).thenReturn(true); + when(oTelTraceSourceConfig.getSslKeyCertChainFile()).thenReturn("data/certificate/test_cert.crt"); + when(oTelTraceSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); + configureObjectUnderTest(); + SOURCE.start(buffer); + + TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + + StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); + + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.UNKNOWN)); + } + @Test void testServerStartCertFileSuccess() throws IOException { try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 4635d4c03d..c44556959e 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -100,14 +100,6 @@ List logTestConfigurations = [ 'otel-logs-source-pipeline.yml', 'data_prepper.yml' ), - new LogTestConfiguration( - 'otelMetricsSourceJsonPayloadEndToEndTest', - 'Runs the basic end-to-end test using the otel metrics source http service.', - 'org.opensearch.dataprepper.integration.metrics.EndToEndOtelMetricsSourceTest.testOtelMetricsSourcePipelineHttpService*', - 'data-prepper-otel-metrics-source', - 'metrics/otel-metrics-source-pipeline.yml', - 'data_prepper.yml' - ), ] diff --git a/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/metrics/EndToEndOtelMetricsSourceTest.java b/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/metrics/EndToEndOtelMetricsSourceTest.java deleted file mode 100644 index 5d3779264b..0000000000 --- a/e2e-test/log/src/integrationTest/java/org/opensearch/dataprepper/integration/metrics/EndToEndOtelMetricsSourceTest.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - */ - -package org.opensearch.dataprepper.integration.metrics; - -import static com.linecorp.armeria.common.MediaType.JSON_UTF_8; -import static com.linecorp.armeria.common.MediaType.PROTOBUF; -import static org.awaitility.Awaitility.await; -import static org.hamcrest.CoreMatchers.everyItem; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.in; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.junit.Test; -import org.opensearch.action.admin.indices.refresh.RefreshRequest; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; -import org.opensearch.client.RequestOptions; -import org.opensearch.client.RestHighLevelClient; -import org.opensearch.dataprepper.plugins.sink.opensearch.ConnectionConfiguration; -import org.opensearch.search.SearchHit; -import org.opensearch.search.SearchHits; -import org.opensearch.search.builder.SearchSourceBuilder; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.WebClient; -import com.linecorp.armeria.common.HttpData; -import com.linecorp.armeria.common.HttpMethod; -import com.linecorp.armeria.common.HttpStatus; -import com.linecorp.armeria.common.MediaType; -import com.linecorp.armeria.common.RequestHeaders; -import com.linecorp.armeria.common.SessionProtocol; - -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; -import io.opentelemetry.proto.common.v1.AnyValue; -import io.opentelemetry.proto.common.v1.InstrumentationScope; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.metrics.v1.Gauge; -import io.opentelemetry.proto.metrics.v1.Metric; -import io.opentelemetry.proto.metrics.v1.NumberDataPoint; -import io.opentelemetry.proto.metrics.v1.ResourceMetrics; -import io.opentelemetry.proto.metrics.v1.ScopeMetrics; -import io.opentelemetry.proto.resource.v1.Resource; - - -public class EndToEndOtelMetricsSourceTest { - private static final int SOURCE_PORT = 2021; - private static final String INDEX_NAME = "otel-metrics-index"; - private static final String HTTP_PATH = "/otel-metrics-pipeline/metrics"; - - final RestHighLevelClient openSearchClient = createOpenSearchClient(); - - @Test - public void testOtelMetricsSourcePipelineHttpServiceEndToEnd() throws InvalidProtocolBufferException { - ingestMetrics(HTTP_PATH, createOtelMetricsJsonRequest(), JSON_UTF_8); - - searchForMetricsAndAssert(); - } - - @Test - public void testOtelMetricsSourcePipelineHttpServiceWithProtobufPayloadEndToEnd() throws InvalidProtocolBufferException { - ingestMetrics(HTTP_PATH, createOtelMetricsProtobufRequest(), PROTOBUF); - - searchForMetricsAndAssert(); - } - - - private HttpData createOtelMetricsJsonRequest() throws InvalidProtocolBufferException { - return HttpData.copyOf(JsonFormat.printer().print(createMetricsServiceRequest()).getBytes()); - } - - private HttpData createOtelMetricsProtobufRequest() throws InvalidProtocolBufferException { - return HttpData.copyOf(createMetricsServiceRequest().toByteArray()); - } - - private void searchForMetricsAndAssert() { - await().atMost(10, TimeUnit.SECONDS).untilAsserted( - () -> { - refreshIndices(); - final SearchRequest searchRequest = new SearchRequest(INDEX_NAME); - searchRequest.source(SearchSourceBuilder.searchSource().size(100)); - final SearchResponse searchResponse = openSearchClient.search(searchRequest, RequestOptions.DEFAULT); - final List> foundMetrics = getMetricsFromSearchHits(searchResponse.getHits()); - assertEquals(1, foundMetrics.size()); - final Map actualMetric = foundMetrics.get(0); - assertThat(createExpectedMetric().entrySet(), everyItem(is(in(actualMetric.entrySet())))); - } - ); - } - - private Map createExpectedMetric() { - return Map.of( - "kind", "GAUGE", - "description", "description", - "serviceName", "service", - "resource.attributes.service@name", "service", - "name", "name", - "value", 4.0 - ); - } - - private RestHighLevelClient createOpenSearchClient() { - return new ConnectionConfiguration.Builder( - Collections.singletonList("https://127.0.0.1:9200")) - .withUsername("admin") - .withPassword("admin") - .withInsecure(true) - .build() - .createClient(null); - } - - private void ingestMetrics(String path, HttpData payload, MediaType mediaType) throws InvalidProtocolBufferException { - RequestHeaders headers = RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority(String.format("127.0.0.1:%d", SOURCE_PORT)) - .method(HttpMethod.POST) - .path(path) - .contentType(mediaType) - .build(); - - WebClient.of().execute(headers, payload) - .aggregate() - .whenComplete((i, ex) -> assertThat(i.status(), is(HttpStatus.OK))) - .join(); - } - - private List> getMetricsFromSearchHits(final SearchHits searchHits) { - return Arrays.stream(searchHits.getHits()) - .map(SearchHit::getSourceAsMap).collect(Collectors.toList()); - } - - private void refreshIndices() throws IOException { - openSearchClient.indices().refresh(new RefreshRequest(), RequestOptions.DEFAULT); - } - - public static ExportMetricsServiceRequest createMetricsServiceRequest() { - final Resource resource = Resource.newBuilder() - .addAttributes(KeyValue.newBuilder() - .setKey("service.name") - .setValue(AnyValue.newBuilder().setStringValue("service").build()) - ).build(); - - NumberDataPoint.Builder p1 = NumberDataPoint.newBuilder().setAsInt(4); - Gauge gauge = Gauge.newBuilder().addDataPoints(p1).build(); - - Metric metric = Metric.newBuilder().setGauge(gauge).setUnit("seconds") - .setName("name") - .setDescription("description") - .build(); - - ScopeMetrics.Builder scopeMetric = ScopeMetrics.newBuilder() - .addMetrics(metric) - .setScope(InstrumentationScope.newBuilder() - .setName("scopeName") - .setVersion("scopeVersion") - .build()); - - ResourceMetrics resourceMetrics = ResourceMetrics.newBuilder() - .setResource(resource) - .addScopeMetrics(scopeMetric) - .build(); - - return ExportMetricsServiceRequest.newBuilder() - .addResourceMetrics(resourceMetrics) - .build(); - } -} diff --git a/e2e-test/log/src/integrationTest/resources/metrics/otel-metrics-source-pipeline.yml b/e2e-test/log/src/integrationTest/resources/metrics/otel-metrics-source-pipeline.yml deleted file mode 100644 index 95bb6bb79a..0000000000 --- a/e2e-test/log/src/integrationTest/resources/metrics/otel-metrics-source-pipeline.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright OpenSearch Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. - -otel-metrics-pipeline: - workers: 2 - source: - otel_metrics_source: - http_path: "/${pipelineName}/metrics" - port: 2021 - ssl: false - sink: - - opensearch: - hosts: [ "https://node-0.example.com:9200" ] - username: "admin" - password: "admin" - insecure: true - index: "otel-metrics-index" - flush_timeout: 5000