From 296596410534c031e142713bab85885b2333af02 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Tue, 5 Nov 2024 15:49:49 +0100 Subject: [PATCH 01/30] Update OpenTelemetryProto to 1.3.2-alpha and refactor scope usage Signed-off-by: Tomas Longo --- .../dataprepper/integration/trace/EndToEndRawSpanTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index 6a8d033572..829b8e8e6b 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -21,7 +21,6 @@ import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.resource.v1.Resource; import io.opentelemetry.proto.trace.v1.ResourceSpans; -import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; import org.opensearch.action.admin.indices.refresh.RefreshRequest; From 975741f198189749fc7dbf558c12fbceb74224a8 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 15 Nov 2024 11:29:09 +0100 Subject: [PATCH 02/30] [WIP] Process ExportTraceServiceRequest in http service Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 126 +- .../source/oteltrace/grpc/GrpcService.java | 196 +++ .../oteltrace/http/ArmeriaHttpService.java | 120 ++ .../source/oteltrace/http/HttpService.java | 27 + .../source/oteltrace/OTelTraceSourceTest.java | 17 +- .../OTelTraceSource_HttpServiceTest.java | 1107 +++++++++++++++++ .../OTelTraceSource_RetryInfoTest.java | 5 +- examples/trace_analytics_no_ssl_2x.yml | 10 +- 8 files changed, 1548 insertions(+), 60 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index a498a63b61..0460c05cdd 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -5,46 +5,47 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.common.util.BlockingTaskExecutor; import com.linecorp.armeria.server.Server; -import io.grpc.MethodDescriptor; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; -import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; -import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; + import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.otel.codec.OTelOutputFormat; -import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoOpensearchCodec; -import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoStandardCodec; -import org.opensearch.dataprepper.plugins.otel.codec.OTelTraceDecoder; -import org.opensearch.dataprepper.plugins.server.CreateServer; -import org.opensearch.dataprepper.plugins.server.ServerConfiguration; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; +import org.opensearch.dataprepper.model.codec.ByteDecoder; +import org.opensearch.dataprepper.plugins.otel.codec.OTelTraceDecoder; +import org.opensearch.dataprepper.plugins.source.oteltrace.grpc.GrpcService; +import org.opensearch.dataprepper.plugins.source.oteltrace.http.HttpService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.util.concurrent.ExecutionException; @DataPrepperPlugin(name = "otel_trace_source", pluginType = Source.class, pluginConfigurationType = OTelTraceSourceConfig.class) public class OTelTraceSource implements Source> { private static final String PLUGIN_NAME = "otel_trace_source"; private static final Logger LOG = LoggerFactory.getLogger(OTelTraceSource.class); + + private static final String HTTP_HEALTH_CHECK_PATH = "/health"; static final String SERVER_CONNECTIONS = "serverConnections"; + private final OTelTraceSourceConfig oTelTraceSourceConfig; private final PluginMetrics pluginMetrics; - private final GrpcAuthenticationProvider authenticationProvider; + private final PluginFactory pluginFactory; private final CertificateProviderFactory certificateProviderFactory; private final String pipelineName; private Server server; @@ -62,9 +63,9 @@ public OTelTraceSource(final OTelTraceSourceConfig oTelTraceSourceConfig, final oTelTraceSourceConfig.validateAndInitializeCertAndKeyFileInS3(); this.oTelTraceSourceConfig = oTelTraceSourceConfig; this.pluginMetrics = pluginMetrics; + this.pluginFactory = pluginFactory; this.certificateProviderFactory = certificateProviderFactory; this.pipelineName = pipelineDescription.getPipelineName(); - this.authenticationProvider = createAuthenticationProvider(pluginFactory); this.byteDecoder = new OTelTraceDecoder(oTelTraceSourceConfig.getOutputFormat()); } @@ -80,33 +81,25 @@ public void start(Buffer> buffer) { } if (server == null) { + ServerBuilder serverBuilder = Server.builder(); + serverBuilder = serverBuilder.port(oTelTraceSourceConfig.getPort(), SessionProtocol.HTTP); - final OTelTraceGrpcService oTelTraceGrpcService = new OTelTraceGrpcService( - (int)(oTelTraceSourceConfig.getRequestTimeoutInMillis() * 0.8), - oTelTraceSourceConfig.getOutputFormat() == OTelOutputFormat.OPENSEARCH ? new OTelProtoOpensearchCodec.OTelProtoDecoder() : new OTelProtoStandardCodec.OTelProtoDecoder(), - buffer, - pluginMetrics - ); + configureHeadersAndHealthCheck(serverBuilder); + configureTLS(serverBuilder); + configureTaskExecutor(serverBuilder); - ServerConfiguration serverConfiguration = ConvertConfiguration.convertConfiguration(oTelTraceSourceConfig); - CreateServer createServer = new CreateServer(serverConfiguration, LOG, pluginMetrics, PLUGIN_NAME, pipelineName); - CertificateProvider certificateProvider = null; - if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { - certificateProvider = certificateProviderFactory.getCertificateProvider(); - } - final MethodDescriptor methodDescriptor = TraceServiceGrpc.getExportMethod(); - server = createServer.createGRPCServer(authenticationProvider, oTelTraceGrpcService, certificateProvider, methodDescriptor); + // todo tlongo convert to factory method? + new GrpcService(pluginFactory, oTelTraceSourceConfig, pluginMetrics, pipelineName, certificateProviderFactory).create(buffer, serverBuilder); + new HttpService(oTelTraceSourceConfig, pluginMetrics).create(serverBuilder, buffer); + + server = serverBuilder.build(); pluginMetrics.gauge(SERVER_CONNECTIONS, server, Server::numConnections); } try { server.start().get(); } catch (ExecutionException ex) { - if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { - throw (RuntimeException) ex.getCause(); - } else { - throw new RuntimeException(ex); - } + handleExecutionException(ex); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new RuntimeException(ex); @@ -114,6 +107,51 @@ public void start(Buffer> buffer) { LOG.info("Started otel_trace_source on port " + oTelTraceSourceConfig.getPort() + "..."); } + private void handleExecutionException(ExecutionException ex) { + if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { + throw (RuntimeException) ex.getCause(); + } else { + throw new RuntimeException(ex); + } + + } + + private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { + serverBuilder.disableServerHeader(); + if (oTelTraceSourceConfig.enableHttpHealthCheck()) { + serverBuilder.service(HTTP_HEALTH_CHECK_PATH, HealthCheckService.builder().longPolling(0).build()); + } + serverBuilder.requestTimeoutMillis(oTelTraceSourceConfig.getRequestTimeoutInMillis()); + if(oTelTraceSourceConfig.getMaxRequestLength() != null) { + serverBuilder.maxRequestLength(oTelTraceSourceConfig.getMaxRequestLength().getBytes()); + } + serverBuilder.maxNumConnections(oTelTraceSourceConfig.getMaxConnectionCount()); + } + + private void configureTLS(ServerBuilder serverBuilder) { + if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { LOG.info("SSL/TLS is enabled."); + final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); + final Certificate certificate = certificateProvider.getCertificate(); + serverBuilder.https(oTelTraceSourceConfig.getPort()).tls( + new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), + new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) + ) + ); + } else { + LOG.warn("Creating otel_trace_source without SSL/TLS. This is not secure."); + LOG.warn("In order to set up TLS for the otel_trace_source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#ssl"); + serverBuilder.http(oTelTraceSourceConfig.getPort()); + } + } + + private void configureTaskExecutor(ServerBuilder serverBuilder) { + final BlockingTaskExecutor blockingTaskExecutor = BlockingTaskExecutor.builder() + .numThreads(oTelTraceSourceConfig.getThreadCount()) + .threadNamePrefix(pipelineName + "-otel_trace") + .build(); + serverBuilder.blockingTaskExecutor(blockingTaskExecutor, true); + } + @Override public void stop() { if (server != null) { @@ -132,22 +170,4 @@ public void stop() { } LOG.info("Stopped otel_trace_source."); } - - private GrpcAuthenticationProvider createAuthenticationProvider(final PluginFactory pluginFactory) { - final PluginModel authenticationConfiguration = oTelTraceSourceConfig.getAuthentication(); - - if (authenticationConfiguration == null || authenticationConfiguration.getPluginName().equals(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { - LOG.warn("Creating otel-trace-source without authentication. This is not secure."); - LOG.warn("In order to set up Http Basic authentication for the otel-trace-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#authentication-configurations"); - } - - final PluginSetting authenticationPluginSetting; - if (authenticationConfiguration != null) { - authenticationPluginSetting = new PluginSetting(authenticationConfiguration.getPluginName(), authenticationConfiguration.getPluginSettings()); - } else { - authenticationPluginSetting = new PluginSetting(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, Collections.emptyMap()); - } - authenticationPluginSetting.setPipelineName(pipelineName); - return pluginFactory.loadPlugin(GrpcAuthenticationProvider.class, authenticationPluginSetting); - } } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java new file mode 100644 index 0000000000..e0ba556b21 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -0,0 +1,196 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace.grpc; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.health.HealthGrpcService; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; +import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; +import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import org.opensearch.dataprepper.plugins.source.oteltrace.RetryInfoConfig; +import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; +import com.linecorp.armeria.common.util.BlockingTaskExecutor; +import com.linecorp.armeria.server.HttpService; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.encoding.DecodingService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; + +import io.grpc.MethodDescriptor; +import io.grpc.ServerInterceptor; +import io.grpc.ServerInterceptors; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; + +public class GrpcService { + private static final Logger LOG = LoggerFactory.getLogger(GrpcService.class); + + // Default RetryInfo with minimum 100ms and maximum 2s + private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); + + private static final String PIPELINE_NAME_PLACEHOLDER = "${pipelineName}"; + private static final String HTTP_HEALTH_CHECK_PATH = "/health"; + public static final String REGEX_HEALTH = "regex:^/(?!health$).*$"; + + private final OTelTraceSourceConfig oTelTraceSourceConfig; + private final GrpcAuthenticationProvider authenticationProvider; + private final PluginMetrics pluginMetrics; + private final String pipelineName; + private final CertificateProviderFactory certificateProviderFactory; + + public GrpcService(PluginFactory pluginFactory, OTelTraceSourceConfig oTelTraceSourceConfig, PluginMetrics pluginMetrics, String pipelineName, CertificateProviderFactory certificateProviderFactory) { + this.oTelTraceSourceConfig = oTelTraceSourceConfig; + this.pluginMetrics = pluginMetrics; + this.pipelineName = pipelineName; + this.authenticationProvider = createAuthenticationProvider(pluginFactory, oTelTraceSourceConfig); + this.certificateProviderFactory = certificateProviderFactory; + } + + public void create(Buffer> buffer, ServerBuilder serverBuilder) { + + final OTelTraceGrpcService oTelTraceGrpcService = new OTelTraceGrpcService( + (int)(oTelTraceSourceConfig.getRequestTimeoutInMillis() * 0.8), + new OTelProtoCodec.OTelProtoDecoder(), + buffer, + pluginMetrics + ); + + final List serverInterceptors = getAuthenticationInterceptor(); + + final GrpcServiceBuilder grpcServiceBuilder = com.linecorp.armeria.server.grpc.GrpcService + .builder() + .useClientTimeoutHeader(false) + .useBlockingTaskExecutor(true) + .exceptionHandler(createGrpExceptionHandler()); + + final MethodDescriptor methodDescriptor = TraceServiceGrpc.getExportMethod(); + final String oTelTraceSourcePath = oTelTraceSourceConfig.getPath(); + if (oTelTraceSourcePath != null) { + final String transformedOTelTraceSourcePath = oTelTraceSourcePath.replace(PIPELINE_NAME_PLACEHOLDER, pipelineName); + grpcServiceBuilder.addService(transformedOTelTraceSourcePath, + ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors), methodDescriptor); + } else { + grpcServiceBuilder.addService(ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors)); + } + + if (oTelTraceSourceConfig.hasHealthCheck()) { + LOG.info("Health check is enabled"); + grpcServiceBuilder.addService(new HealthGrpcService()); + } + + if (oTelTraceSourceConfig.hasProtoReflectionService()) { + LOG.info("Proto reflection service is enabled"); + grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); + } + + // todo tlongo let this method return the grpc service. All things serverbuilder related have to be done by the source + + grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); + +// serverBuilder.disableServerHeader(); + if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { + serverBuilder.service(grpcServiceBuilder.build()); + } else { + serverBuilder.service(grpcServiceBuilder.build(), DecodingService.newDecorator()); + } + +// if (oTelTraceSourceConfig.enableHttpHealthCheck()) { +// serverBuilder.service(HTTP_HEALTH_CHECK_PATH, HealthCheckService.builder().longPolling(0).build()); +// } + + if (oTelTraceSourceConfig.getAuthentication() != null) { + final Optional> optionalHttpAuthenticationService = + authenticationProvider.getHttpAuthenticationService(); + + if (oTelTraceSourceConfig.isUnauthenticatedHealthCheck()) { + optionalHttpAuthenticationService.ifPresent(httpAuthenticationService -> + serverBuilder.decorator(REGEX_HEALTH, httpAuthenticationService)); + } else { + optionalHttpAuthenticationService.ifPresent(serverBuilder::decorator); + } + } + +// serverBuilder.requestTimeoutMillis(oTelTraceSourceConfig.getRequestTimeoutInMillis()); +// if(oTelTraceSourceConfig.getMaxRequestLength() != null) { +// serverBuilder.maxRequestLength(oTelTraceSourceConfig.getMaxRequestLength().getBytes()); +// } + + // ACM Cert for SSL takes preference +// if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { LOG.info("SSL/TLS is enabled."); +// final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); +// final Certificate certificate = certificateProvider.getCertificate(); +// serverBuilder.https(oTelTraceSourceConfig.getPort()).tls( +// new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), +// new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) +// ) +// ); +// } else { +// LOG.warn("Creating otel_trace_source without SSL/TLS. This is not secure."); +// LOG.warn("In order to set up TLS for the otel_trace_source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#ssl"); +// serverBuilder.http(oTelTraceSourceConfig.getPort()); +// } + +// serverBuilder.maxNumConnections(oTelTraceSourceConfig.getMaxConnectionCount()); +// final BlockingTaskExecutor blockingTaskExecutor = BlockingTaskExecutor.builder() +// .numThreads(oTelTraceSourceConfig.getThreadCount()) +// .threadNamePrefix(pipelineName + "-otel_trace") +// .build(); +// serverBuilder.blockingTaskExecutor(blockingTaskExecutor, true); + } + + private List getAuthenticationInterceptor() { + final ServerInterceptor authenticationInterceptor = authenticationProvider.getAuthenticationInterceptor(); + if (authenticationInterceptor == null) { + return Collections.emptyList(); + } + return Collections.singletonList(authenticationInterceptor); + } + + private GrpcAuthenticationProvider createAuthenticationProvider(final PluginFactory pluginFactory, final OTelTraceSourceConfig oTelTraceSourceConfig) { + final PluginModel authenticationConfiguration = oTelTraceSourceConfig.getAuthentication(); + + if (authenticationConfiguration == null || authenticationConfiguration.getPluginName().equals(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { + LOG.warn("Creating otel-trace-source without authentication. This is not secure."); + LOG.warn("In order to set up Http Basic authentication for the otel-trace-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#authentication-configurations"); + } + + final PluginSetting authenticationPluginSetting; + if (authenticationConfiguration != null) { + authenticationPluginSetting = new PluginSetting(authenticationConfiguration.getPluginName(), authenticationConfiguration.getPluginSettings()); + } else { + authenticationPluginSetting = new PluginSetting(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, Collections.emptyMap()); + } + authenticationPluginSetting.setPipelineName(pipelineName); + return pluginFactory.loadPlugin(GrpcAuthenticationProvider.class, authenticationPluginSetting); + } + + private GrpcExceptionHandlerFunction createGrpExceptionHandler() { + RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null + ? oTelTraceSourceConfig.getRetryInfo() + : DEFAULT_RETRY_INFO; + + return new GrpcRequestExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java new file mode 100644 index 0000000000..aa7dbe20ce --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -0,0 +1,120 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace.http; + +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.opensearch.dataprepper.exceptions.BadRequestException; +import org.opensearch.dataprepper.exceptions.BufferWriteException; +import org.opensearch.dataprepper.logging.DataPrepperMarkers; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.trace.Span; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; +import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.annotation.Consumes; +import com.linecorp.armeria.server.annotation.Post; + +import io.grpc.stub.StreamObserver; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Timer; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; + +public class ArmeriaHttpService { + private static final Logger LOG = LoggerFactory.getLogger(ArmeriaHttpService.class); + + public static final String REQUEST_TIMEOUTS = "requestTimeouts"; + public static final String REQUESTS_RECEIVED = "requestsReceived"; + public static final String BAD_REQUESTS = "badRequests"; + public static final String REQUESTS_TOO_LARGE = "requestsTooLarge"; + public static final String INTERNAL_SERVER_ERROR = "internalServerError"; + public static final String SUCCESS_REQUESTS = "successRequests"; + public static final String PAYLOAD_SIZE = "payloadSize"; + public static final String REQUEST_PROCESS_DURATION = "requestProcessDuration"; + + final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder; + final Buffer> buffer; + // todo tlongo config + private int bufferWriteTimeoutInMillis = 10_000; + + private final Counter requestsReceivedCounter; + private final Counter successRequestsCounter; + private final DistributionSummary payloadSizeSummary; + private final Timer requestProcessDuration; + + public ArmeriaHttpService(Buffer> buffer, final PluginMetrics pluginMetrics) { + this.buffer = buffer; + this.oTelProtoDecoder = new OTelProtoCodec.OTelProtoDecoder(); + + // todo tlongo encapsulate into own class, since both, grpc and http, should contribute to those + requestsReceivedCounter = pluginMetrics.counter(REQUESTS_RECEIVED); + successRequestsCounter = pluginMetrics.counter(SUCCESS_REQUESTS); + payloadSizeSummary = pluginMetrics.summary(PAYLOAD_SIZE); + requestProcessDuration = pluginMetrics.timer(REQUEST_PROCESS_DURATION); + } + + // todo tlongo handle excpetions + // todo tlongo handle backoff + // todo tlongo healthcheck? + + @Post("/hello") + @Consumes(value = "application/json") + public void hello(ExportTraceServiceRequest request) { + requestsReceivedCounter.increment(); + payloadSizeSummary.record(request.getSerializedSize()); + + requestProcessDuration.record(() -> processRequest(request)); + } + + // todo tlongo exract in order to be used by http and grpc? + private void processRequest(final ExportTraceServiceRequest request) { + final Collection spans; + + try { + spans = oTelProtoDecoder.parseExportTraceServiceRequest(request, Instant.now()); + } catch (final Exception e) { + LOG.warn(DataPrepperMarkers.SENSITIVE, "Failed to parse request with error '{}'. Request body: {}.", e.getMessage(), request); + throw new BadRequestException(e.getMessage(), e); + } + + try { + if (buffer.isByteBuffer()) { + Map requestsMap = oTelProtoDecoder.splitExportTraceServiceRequestByTraceId(request); + for (Map.Entry entry: requestsMap.entrySet()) { + buffer.writeBytes(entry.getValue().toByteArray(), entry.getKey(), bufferWriteTimeoutInMillis); + } + } else { + final List> records = spans.stream().map(span -> new Record(span)).collect(Collectors.toList()); + buffer.writeAll(records, bufferWriteTimeoutInMillis); + } + } catch (final Exception e) { + if (ServiceRequestContext.current().isTimedOut()) { + LOG.warn("Exception writing to buffer but request already timed out.", e); + return; + } + + LOG.error("Failed to write the request of size {} due to:", request.toString().length(), e); + throw new BufferWriteException(e.getMessage(), e); + } + + if (ServiceRequestContext.current().isTimedOut()) { + LOG.warn("Buffer write completed successfully but request already timed out."); + return; + } + + successRequestsCounter.increment(); + + // todo tlongo what is the responseObserver used for? +// responseObserver.onNext(ExportTraceServiceResponse.newBuilder().build()); +// responseObserver.onCompleted(); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java new file mode 100644 index 0000000000..eedc09a73c --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java @@ -0,0 +1,27 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace.http; + +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.server.ServerBuilder; + +public class HttpService { + private static final Logger LOG = LoggerFactory.getLogger(HttpService.class); + private final OTelTraceSourceConfig oTelTraceSourceConfig; + private final PluginMetrics pluginMetrics; + + public HttpService(OTelTraceSourceConfig oTelTraceSourceConfig, final PluginMetrics pluginMetrics) { + this.oTelTraceSourceConfig = oTelTraceSourceConfig; + this.pluginMetrics = pluginMetrics; + } + + public void create(ServerBuilder serverBuilder, Buffer> buffer) { + // todo tlongo what about tls? + LOG.info("Creating http service"); + serverBuilder.annotatedService(new ArmeriaHttpService(buffer, pluginMetrics)); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 39ef2a9b42..78dbf83bf7 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -52,6 +52,8 @@ import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -120,6 +122,7 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doThrow; @@ -191,6 +194,7 @@ class OTelTraceSourceTest { @Mock private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; + private PluginSetting pluginSetting; private PluginSetting testPluginSetting; private PluginMetrics pluginMetrics; @@ -199,11 +203,13 @@ class OTelTraceSourceTest { @BeforeEach void beforeEach() { + lenient().when(serverBuilder.port(anyInt(), ArgumentMatchers.any())).thenReturn(serverBuilder); lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); lenient().when(serverBuilder.service(any(GrpcService.class), any(Function.class))).thenReturn(serverBuilder); lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); lenient().when(serverBuilder.build()).thenReturn(server); + lenient().when(server.start()).thenReturn(completableFuture); lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); @@ -223,7 +229,7 @@ void beforeEach() { when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) .thenReturn(authenticationProvider); configureObjectUnderTest(); pipelineDescription = mock(PipelineDescription.class); @@ -244,6 +250,8 @@ private void configureObjectUnderTest() { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } + + @Test void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { configureObjectUnderTest(); @@ -1271,6 +1279,13 @@ private ExportTraceServiceRequest createExportTraceRequest() { .build(); } + private void assertJsonResponse(final String expectedResponseBody, final AggregatedHttpResponse response) { + String body = response.content(StandardCharsets.UTF_8); + + assertThat(body, is(expectedResponseBody)); + } + + private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, final HttpStatus expectedStatus, final Throwable throwable) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java new file mode 100644 index 0000000000..b9b0661149 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -0,0 +1,1107 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.oteltrace; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.StringJoiner; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; +import org.opensearch.dataprepper.metrics.MetricNames; +import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.types.ByteCount; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.health.HealthGrpcService; +import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.server.HttpService; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; + +import io.grpc.BindableService; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.instrument.Measurement; +import io.micrometer.core.instrument.Statistic; +import io.netty.util.AsciiString; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + +@ExtendWith(MockitoExtension.class) +class OTelTraceSource_HttpServiceTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; + private static final String USERNAME = "test_user"; + private static final String PASSWORD = "test_password"; + private static final String TEST_PATH = "${pipelineName}/v1/traces"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); + private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() + .addSpans(Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); + private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() + .addSpans(Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); + + @Mock + private ServerBuilder serverBuilder; + + @Mock + private Server server; + + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + + @Mock + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; + + @Mock + private Certificate certificate; + + @Mock + private CompletableFuture completableFuture; + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelTraceSourceConfig oTelTraceSourceConfig; + + @Mock + private Buffer> buffer; + + @Mock + private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; + + @Captor + ArgumentCaptor bytesCaptor; + + private PluginSetting pluginSetting; + private PluginSetting testPluginSetting; + private PluginMetrics pluginMetrics; + private PipelineDescription pipelineDescription; + private OTelTraceSource SOURCE; + + @BeforeEach + void beforeEach() { + lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.service(any(GrpcService.class), any(Function.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.build()).thenReturn(server); + lenient().when(server.start()).thenReturn(completableFuture); + + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + + when(oTelTraceSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelTraceSourceConfig.isSsl()).thenReturn(false); + when(oTelTraceSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelTraceSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelTraceSourceConfig.getThreadCount()).thenReturn(5); + when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + configureObjectUnderTest(); + pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + MetricsTestUtil.initMetrics(); + pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); + + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + + @Test + void testHttpService() throws Exception { + when(buffer.isByteBuffer()).thenReturn(true); + ExportTraceServiceRequest request = createExportTraceRequest(); + + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/hello") + .contentType(MediaType.JSON_UTF_8) + .build(), HttpData.copyOf(JsonFormat.printer().print(request).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertJsonResponse("", response)) + .join(); + + verify(buffer, times(1)).writeBytes(bytesCaptor.capture(), anyString(), anyInt()); + } + + + + + + + + + + + @Test + void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpsFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put("request_timeout", 5); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + pluginSetting = new PluginSetting("otel_trace", settingsMap); + pluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelTraceSourceConfig.class); + SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + + SOURCE.start(buffer); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullBytesWithNonUnframedRequests() { + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(SUCCESS_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(FAILURE_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithUnframedRequests() throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpCompressionWithUnframedRequests() throws IOException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.GZIP); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") + .build(), + createGZipCompressedPayload(JsonFormat.printer().print(createExportTraceRequest()))) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(transformedPath) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndAuthHeader_with_successful_response() throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String encodeToString = Base64.getEncoder() + .encodeToString(String.format("%s:%s", USERNAME, PASSWORD).getBytes(StandardCharsets.UTF_8)); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + + WebClient.of().prepare() + .post("http://127.0.0.1:21890" + transformedPath) + .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportTraceRequest()).getBytes()) + .header("Authorization", "Basic " + encodeToString) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndAuthHeader_with_unsuccessful_response() throws InvalidProtocolBufferException { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + + WebClient.of().prepare() + .post("http://127.0.0.1:21890" + transformedPath) + .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportTraceRequest()).getBytes()) + .execute() + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) + .join(); + } + + @Test + void testServerStartCertFileSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate, is(certAsString)); + assertThat(actualPrivateKey, is(keyAsString)); + } + } + + @Test + void testServerStartACMCertSuccess() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + + final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); + final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); + verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); + final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); + final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); + assertThat(actualCertificate, is(certAsString)); + assertThat(actualPrivateKey, is(keyAsString)); + } + } + + @Test + void start_with_Health_configured_includes_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + @Test + void start_with_Health_configured_unframed_requests_includes_HTTPHealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder).service(eq("/health"), isA(HealthCheckService.class)); + } + + @Test + void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + // todo tlongo remove everything related to unframed requests? + @Test + void start_without_Health_configured_unframed_requests_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + @Test + void testHealthCheckUnauthNotAllowed() { + // Prepare + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, false); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + settingsMap.put("proto_reflection_service", "true"); + settingsMap.put("unauthenticated_health_check", "false"); + settingsMap.put("authentication", new PluginModel("http_basic", + Map.of( + "username", "test", + "password", "test2" + ))); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + + source.start(buffer); + + // When + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("localhost:21890") + .method(HttpMethod.GET) + .path("/health") + .build()) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) + .join(); + + source.stop(); + } + + @Test + void testHealthCheckUnauthAllowed() { + // Prepare + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, false); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + settingsMap.put("proto_reflection_service", "true"); + settingsMap.put("unauthenticated_health_check", "true"); + settingsMap.put("authentication", new PluginModel("http_basic", + Map.of( + "username", "test", + "password", "test2" + ))); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + + source.start(buffer); + + // When + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("localhost:21890") + .method(HttpMethod.GET) + .path("/health") + .build()) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)).join(); + + source.stop(); + } + + @Test + void testOptionalHttpAuthServiceNotInPlace() { + when(server.stop()).thenReturn(completableFuture); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + SOURCE.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder, never()).decorator(isA(Function.class)); + } + + @Test + void testOptionalHttpAuthServiceInPlace() { + final Optional> function = Optional.of(httpService -> httpService); + + final Map settingsMap = new HashMap<>(); + settingsMap.put("authentication", new PluginModel("test", null)); + settingsMap.put("unauthenticated_health_check", true); + + settingsMap.put(SSL, false); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + + when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); + + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder).decorator(isA(String.class), isA(Function.class)); + } + + @Test + void testOptionalHttpAuthServiceInPlaceWithUnauthenticatedDisabled() { + final Optional> function = Optional.of(httpService -> httpService); + + final Map settingsMap = new HashMap<>(); + settingsMap.put("authentication", new PluginModel("test", null)); + settingsMap.put("unauthenticated_health_check", false); + + settingsMap.put(SSL, false); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + + when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); + + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + + try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + } + + verify(serverBuilder).service(isA(GrpcService.class)); + verify(serverBuilder).decorator(isA(Function.class)); + } + + @Test + void testDoubleStart() { + // starting server + SOURCE.start(buffer); + // double start server + assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); + } + + @Test + void testRunAnotherSourceWithSamePort() { + // starting server + SOURCE.start(buffer); + + + Map settingsMap = Map.of("retry_info", TEST_RETRY_INFO, SSL, false); + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + //Expect RuntimeException because when port is already in use, BindException is thrown which is not RuntimeException + assertThrows(RuntimeException.class, () -> source.start(buffer)); + } + + @Test + void testStartWithEmptyBuffer() { + testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + assertThrows(IllegalStateException.class, () -> source.start(null)); + } + + @Test + void testStartWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(completableFuture.get()).thenThrow(new ExecutionException("", null)); + + // When/Then + assertThrows(RuntimeException.class, () -> source.start(buffer)); + } + } + + @Test + void testStartWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + final NullPointerException expCause = new NullPointerException(); + when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); + + // When/Then + final RuntimeException ex = assertThrows(RuntimeException.class, () -> source.start(buffer)); + assertEquals(expCause, ex); + } + } + + @Test + void testStopWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + + // When/Then + when(completableFuture.get()).thenThrow(new ExecutionException("", null)); + assertThrows(RuntimeException.class, source::stop); + } + } + + @Test + void testStartWithInterruptedException() throws ExecutionException, InterruptedException { + // Prepare + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + when(completableFuture.get()).thenThrow(new InterruptedException()); + + // When/Then + assertThrows(RuntimeException.class, () -> source.start(buffer)); + assertTrue(Thread.interrupted()); + } + } + + @Test + void testStopWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { + // Prepare + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + final NullPointerException expCause = new NullPointerException(); + when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); + + // When/Then + final RuntimeException ex = assertThrows(RuntimeException.class, source::stop); + assertEquals(expCause, ex); + } + } + + @Test + void testStopWithInterruptedException() throws ExecutionException, InterruptedException { + // Prepare + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + source.start(buffer); + when(server.stop()).thenReturn(completableFuture); + when(completableFuture.get()).thenThrow(new InterruptedException()); + + // When/Then + assertThrows(RuntimeException.class, source::stop); + assertTrue(Thread.interrupted()); + } + } + + @Test + void request_that_exceeds_maxRequestLength_returns_413() throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getMaxRequestLength()).thenReturn(ByteCount.ofBytes(4)); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) + .join(); + } + + @Test + void testServerConnectionsMetric() throws InvalidProtocolBufferException { + // Prepare + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + SOURCE.start(buffer); + + final String metricNamePrefix = new StringJoiner(MetricNames.DELIMITER) + .add("pipeline").add("otel_trace").toString(); + List serverConnectionsMeasurements = MetricsTestUtil.getMeasurementList( + new StringJoiner(MetricNames.DELIMITER).add(metricNamePrefix) + .add(OTelTraceSource.SERVER_CONNECTIONS).toString()); + + // Verify connections metric value is 0 + Measurement serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); + assertEquals(0, serverConnectionsMeasurement.getValue()); + + final RequestHeaders testRequestHeaders = RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(); + final HttpData testHttpData = HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes()); + + // Send request + WebClient.of().execute(testRequestHeaders, testHttpData) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + + // Verify connections metric value is 1 + serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); + assertEquals(1.0, serverConnectionsMeasurement.getValue()); + } + + private ExportTraceServiceRequest createExportTraceRequest() { + final Span testSpan = Span.newBuilder() + .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + } + + private void assertJsonResponse(final String expectedResponseBody, final AggregatedHttpResponse response) { + String body = response.content(StandardCharsets.UTF_8); + + assertThat(body, is(expectedResponseBody)); + } + + + private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, + final HttpStatus expectedStatus, + final Throwable throwable) { + assertThat("Http Status", response.status(), equalTo(expectedStatus)); + assertThat("Http Response Throwable", throwable, is(nullValue())); + + final List headerKeys = response.headers() + .stream() + .map(Map.Entry::getKey) + .map(AsciiString::toString) + .collect(Collectors.toList()); + assertThat("Response Header Keys", headerKeys, not(contains("server"))); + } + + private byte[] createGZipCompressedPayload(final String payload) throws IOException { + // Create a GZip compressed request body + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { + gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); + } + return byteStream.toByteArray(); + } + +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java index e2613d5dc8..4d59e53800 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_RetryInfoTest.java @@ -155,8 +155,9 @@ private ExportTraceServiceRequest createExportTraceRequest() { .setEndTimeUnixNano(101) .setTraceState("SUCCESS").build(); - ScopeSpans scopeSpan = ScopeSpans.newBuilder().addSpans(testSpan).build(); return ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder().addScopeSpans(scopeSpan)).build(); + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); } } diff --git a/examples/trace_analytics_no_ssl_2x.yml b/examples/trace_analytics_no_ssl_2x.yml index a1ce76575a..23d7e056bb 100644 --- a/examples/trace_analytics_no_ssl_2x.yml +++ b/examples/trace_analytics_no_ssl_2x.yml @@ -16,10 +16,11 @@ raw-pipeline: - otel_traces: sink: - opensearch: + insecure: true hosts: [ "https://node-0.example.com:9200" ] - cert: "/usr/share/data-prepper/root-ca.pem" +# cert: "/usr/share/data-prepper/root-ca.pem" username: "admin" - password: "admin" + password: "M__!!ega123!" index_type: trace-analytics-raw service-map-pipeline: delay: "100" @@ -30,8 +31,9 @@ service-map-pipeline: - service_map: sink: - opensearch: + insecure: true hosts: ["https://node-0.example.com:9200"] - cert: "/usr/share/data-prepper/root-ca.pem" +# cert: "/usr/share/data-prepper/root-ca.pem" username: "admin" - password: "admin" + password: "M__!!ega123!" index_type: trace-analytics-service-map From cfe31d9be53475387f7d1fe6422f706915f7195c Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 29 Nov 2024 13:42:42 +0100 Subject: [PATCH 03/30] [WIP] Move gRPC request tests to own class Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSourceTest.java | 131 ------ .../OTelTraceSource_GrpcRequestTest.java | 418 ++++++++++++++++++ .../OTelTraceSource_HttpServiceTest.java | 1 - .../docker-compose.yml | 3 +- 4 files changed, 420 insertions(+), 133 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 78dbf83bf7..40587c398c 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -11,7 +11,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import com.linecorp.armeria.client.ClientFactory; -import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.ClosedSessionException; @@ -31,13 +30,10 @@ import io.grpc.BindableService; import io.grpc.ServerServiceDefinition; import io.grpc.Status; -import io.grpc.StatusRuntimeException; import io.micrometer.core.instrument.Measurement; import io.micrometer.core.instrument.Statistic; import io.netty.util.AsciiString; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; -import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; @@ -47,13 +43,10 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; -import org.mockito.Captor; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -88,7 +81,6 @@ import java.nio.file.Path; import java.time.Duration; import java.util.Base64; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -109,10 +101,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -120,18 +110,14 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; @@ -139,7 +125,6 @@ @ExtendWith(MockitoExtension.class) class OTelTraceSourceTest { - private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; private static final String USERNAME = "test_user"; private static final String PASSWORD = "test_password"; private static final String TEST_PATH = "${pipelineName}/v1/traces"; @@ -194,7 +179,6 @@ class OTelTraceSourceTest { @Mock private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; - private PluginSetting pluginSetting; private PluginSetting testPluginSetting; private PluginMetrics pluginMetrics; @@ -250,8 +234,6 @@ private void configureObjectUnderTest() { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } - - @Test void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { configureObjectUnderTest(); @@ -1078,112 +1060,6 @@ void testStopWithInterruptedException() throws ExecutionException, InterruptedEx } } - @Test - void gRPC_request_writes_to_buffer_with_successful_response() throws Exception { - configureObjectUnderTest(); - SOURCE.start(buffer); - - final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .build(TraceServiceGrpc.TraceServiceBlockingStub.class); - final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); - assertThat(exportResponse, notNullValue()); - - final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); - verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); - - final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites, hasSize(1)); - } - - @Test - void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws Exception { - when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); - when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); - final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); - - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) - .thenReturn(grpcAuthenticationProvider); - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", - Map.of( - "username", USERNAME, - "password", PASSWORD - ))); - configureObjectUnderTest(); - SOURCE.start(buffer); - - final String encodeToString = Base64.getEncoder() - .encodeToString(String.format("%s:%s", USERNAME, PASSWORD).getBytes(StandardCharsets.UTF_8)); - - final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .addHeader("Authorization", "Basic " + encodeToString) - .build(TraceServiceGrpc.TraceServiceBlockingStub.class); - final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); - assertThat(exportResponse, notNullValue()); - - final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); - verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); - - final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); - assertThat(actualBufferWrites, notNullValue()); - assertThat(actualBufferWrites, hasSize(1)); - } - - @Test - void gRPC_request_with_custom_path_throws_when_written_to_default_path() { - when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - - configureObjectUnderTest(); - SOURCE.start(buffer); - - final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .build(TraceServiceGrpc.TraceServiceBlockingStub.class); - - final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); - assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); - } - - @ParameterizedTest - @ArgumentsSource(BufferExceptionToStatusArgumentsProvider.class) - void gRPC_request_returns_expected_status_for_exceptions_from_buffer( - final Class bufferExceptionClass, - final Status.Code expectedStatusCode) throws Exception { - configureObjectUnderTest(); - SOURCE.start(buffer); - - final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .build(TraceServiceGrpc.TraceServiceBlockingStub.class); - - doThrow(bufferExceptionClass) - .when(buffer) - .writeAll(anyCollection(), anyInt()); - final ExportTraceServiceRequest exportTraceRequest = createExportTraceRequest(); - final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportTraceRequest)); - - assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(expectedStatusCode)); - } - - @Test - void gRPC_request_throws_InvalidArgument_for_malformed_trace_data() { - configureObjectUnderTest(); - SOURCE.start(buffer); - - final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) - .build(TraceServiceGrpc.TraceServiceBlockingStub.class); - - final ExportTraceServiceRequest exportTraceRequest = createInvalidExportTraceRequest(); - final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportTraceRequest)); - - assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.INVALID_ARGUMENT)); - - verifyNoInteractions(buffer); - } - @Test void request_that_exceeds_maxRequestLength_returns_413() throws InvalidProtocolBufferException { when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); @@ -1279,13 +1155,6 @@ private ExportTraceServiceRequest createExportTraceRequest() { .build(); } - private void assertJsonResponse(final String expectedResponseBody, final AggregatedHttpResponse response) { - String body = response.content(StandardCharsets.UTF_8); - - assertThat(body, is(expectedResponseBody)); - } - - private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, final HttpStatus expectedStatus, final Throwable throwable) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java new file mode 100644 index 0000000000..64bf03e38e --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java @@ -0,0 +1,418 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.oteltrace; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.StringJoiner; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; +import org.opensearch.dataprepper.metrics.MetricNames; +import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.types.ByteCount; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.health.HealthGrpcService; +import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.Clients; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.server.HttpService; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; + +import io.grpc.BindableService; +import io.grpc.ServerServiceDefinition; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.micrometer.core.instrument.Measurement; +import io.micrometer.core.instrument.Statistic; +import io.netty.util.AsciiString; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + +@ExtendWith(MockitoExtension.class) +class OTelTraceSource_GrpcRequestTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; + private static final String USERNAME = "test_user"; + private static final String PASSWORD = "test_password"; + private static final String TEST_PATH = "${pipelineName}/v1/traces"; + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); + + @Mock + private ServerBuilder serverBuilder; + + @Mock + private Server server; + + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + + @Mock + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; + + @Mock + private Certificate certificate; + + @Mock + private CompletableFuture completableFuture; + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelTraceSourceConfig oTelTraceSourceConfig; + + @Mock + private Buffer> buffer; + + @Mock + private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; + + + private PluginMetrics pluginMetrics; + private PipelineDescription pipelineDescription; + private OTelTraceSource SOURCE; + + @BeforeEach + void beforeEach() { + lenient().when(serverBuilder.port(anyInt(), ArgumentMatchers.any())).thenReturn(serverBuilder); + lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.service(any(GrpcService.class), any(Function.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.build()).thenReturn(server); + + lenient().when(server.start()).thenReturn(completableFuture); + + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + + when(oTelTraceSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelTraceSourceConfig.isSsl()).thenReturn(false); + when(oTelTraceSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelTraceSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelTraceSourceConfig.getThreadCount()).thenReturn(5); + when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + configureObjectUnderTest(); + pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + MetricsTestUtil.initMetrics(); + pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); + + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + + + @Test + void gRPC_request_writes_to_buffer_with_successful_response() throws Exception { + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); + assertThat(exportResponse, notNullValue()); + + final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); + verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); + + final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); + assertThat(actualBufferWrites, notNullValue()); + assertThat(actualBufferWrites, hasSize(1)); + } + + @Test + void gRPC_with_auth_request_writes_to_buffer_with_successful_response() throws Exception { + when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); + when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); + final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); + + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(grpcAuthenticationProvider); + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", + Map.of( + "username", USERNAME, + "password", PASSWORD + ))); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String encodeToString = Base64.getEncoder() + .encodeToString(String.format("%s:%s", USERNAME, PASSWORD).getBytes(StandardCharsets.UTF_8)); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .addHeader("Authorization", "Basic " + encodeToString) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + final ExportTraceServiceResponse exportResponse = client.export(createExportTraceRequest()); + assertThat(exportResponse, notNullValue()); + + final ArgumentCaptor>> bufferWriteArgumentCaptor = ArgumentCaptor.forClass(Collection.class); + verify(buffer).writeAll(bufferWriteArgumentCaptor.capture(), anyInt()); + + final Collection> actualBufferWrites = bufferWriteArgumentCaptor.getValue(); + assertThat(actualBufferWrites, notNullValue()); + assertThat(actualBufferWrites, hasSize(1)); + } + + @Test + void gRPC_request_with_custom_path_throws_when_written_to_default_path() { + when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); + } + + @ParameterizedTest + @ArgumentsSource(BufferExceptionToStatusArgumentsProvider.class) + void gRPC_request_returns_expected_status_for_exceptions_from_buffer( + final Class bufferExceptionClass, + final Status.Code expectedStatusCode) throws Exception { + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + + doThrow(bufferExceptionClass) + .when(buffer) + .writeAll(anyCollection(), anyInt()); + final ExportTraceServiceRequest exportTraceRequest = createExportTraceRequest(); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportTraceRequest)); + + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(expectedStatusCode)); + } + + @Test + void gRPC_request_throws_InvalidArgument_for_malformed_trace_data() { + configureObjectUnderTest(); + SOURCE.start(buffer); + + final TraceServiceGrpc.TraceServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) + .build(TraceServiceGrpc.TraceServiceBlockingStub.class); + + final ExportTraceServiceRequest exportTraceRequest = createInvalidExportTraceRequest(); + final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportTraceRequest)); + + assertThat(actualException.getStatus(), notNullValue()); + assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.INVALID_ARGUMENT)); + + verifyNoInteractions(buffer); + } + + @Test + void request_that_exceeds_maxRequestLength_returns_413() throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getMaxRequestLength()).thenReturn(ByteCount.ofBytes(4)); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) + .join(); + } + + + static class BufferExceptionToStatusArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext context) { + return Stream.of( + arguments(TimeoutException.class, Status.Code.RESOURCE_EXHAUSTED), + arguments(SizeOverflowException.class, Status.Code.RESOURCE_EXHAUSTED), + arguments(Exception.class, Status.Code.INTERNAL), + arguments(RuntimeException.class, Status.Code.INTERNAL) + ); + } + } + + private ExportTraceServiceRequest createInvalidExportTraceRequest() { + final Span testSpan = Span.newBuilder() + .setTraceState("SUCCESS").build(); + final ExportTraceServiceRequest successRequest = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + + return successRequest; + } + + private ExportTraceServiceRequest createExportTraceRequest() { + final Span testSpan = Span.newBuilder() + .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + } + + private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, + final HttpStatus expectedStatus, + final Throwable throwable) { + assertThat("Http Status", response.status(), equalTo(expectedStatus)); + assertThat("Http Response Throwable", throwable, is(nullValue())); + + final List headerKeys = response.headers() + .stream() + .map(Map.Entry::getKey) + .map(AsciiString::toString) + .collect(Collectors.toList()); + assertThat("Response Header Keys", headerKeys, not(contains("server"))); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index b9b0661149..ae5b1939e9 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -136,7 +136,6 @@ @ExtendWith(MockitoExtension.class) class OTelTraceSource_HttpServiceTest { - private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; private static final String USERNAME = "test_user"; private static final String PASSWORD = "test_password"; private static final String TEST_PATH = "${pipelineName}/v1/traces"; diff --git a/examples/trace-analytics-sample-app/docker-compose.yml b/examples/trace-analytics-sample-app/docker-compose.yml index d0522ee1ac..7d245f9a1e 100644 --- a/examples/trace-analytics-sample-app/docker-compose.yml +++ b/examples/trace-analytics-sample-app/docker-compose.yml @@ -3,7 +3,7 @@ services: data-prepper: restart: unless-stopped container_name: data-prepper - image: opensearchproject/data-prepper:2 + image: opensearch-data-prepper:2.11.0-SNAPSHOT volumes: - ../trace_analytics_no_ssl_2x.yml:/usr/share/data-prepper/pipelines/pipelines.yaml - ../data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml @@ -21,6 +21,7 @@ services: - discovery.type=single-node - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=M__!!ega123!" ulimits: memlock: soft: -1 From 495edf80212dd54aa1cdefd06b21e7720b1a706d Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 29 Nov 2024 13:47:07 +0100 Subject: [PATCH 04/30] [WIP] Cleanup Signed-off-by: Tomas Longo --- .../OTelTraceSource_HttpServiceTest.java | 911 +----------------- 1 file changed, 1 insertion(+), 910 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index ae5b1939e9..cd659fe539 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -6,150 +6,70 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; -import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; -import java.util.Base64; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.StringJoiner; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.zip.GZIPOutputStream; -import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; -import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.model.types.ByteCount; import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; -import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.health.HealthGrpcService; -import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.ClientFactory; -import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpData; -import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpMethod; -import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; -import com.linecorp.armeria.server.healthcheck.HealthCheckService; import io.grpc.BindableService; -import io.grpc.ServerServiceDefinition; -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.micrometer.core.instrument.Measurement; -import io.micrometer.core.instrument.Statistic; -import io.netty.util.AsciiString; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; -import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; -import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; @ExtendWith(MockitoExtension.class) class OTelTraceSource_HttpServiceTest { - private static final String USERNAME = "test_user"; - private static final String PASSWORD = "test_password"; - private static final String TEST_PATH = "${pipelineName}/v1/traces"; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); - private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .addScopeSpans(ScopeSpans.newBuilder() - .addSpans(Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); - private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .addScopeSpans(ScopeSpans.newBuilder() - .addSpans(Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); @Mock private ServerBuilder serverBuilder; @@ -163,15 +83,6 @@ class OTelTraceSource_HttpServiceTest { @Mock private GrpcService grpcService; - @Mock - private CertificateProviderFactory certificateProviderFactory; - - @Mock - private CertificateProvider certificateProvider; - - @Mock - private Certificate certificate; - @Mock private CompletableFuture completableFuture; @@ -187,14 +98,9 @@ class OTelTraceSource_HttpServiceTest { @Mock private Buffer> buffer; - @Mock - private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; - @Captor ArgumentCaptor bytesCaptor; - private PluginSetting pluginSetting; - private PluginSetting testPluginSetting; private PluginMetrics pluginMetrics; private PipelineDescription pipelineDescription; private OTelTraceSource SOURCE; @@ -267,798 +173,8 @@ void testHttpService() throws Exception { verify(buffer, times(1)).writeBytes(bytesCaptor.capture(), anyString(), anyInt()); } - - - - - - - - - - @Test - void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { - configureObjectUnderTest(); - SOURCE.start(buffer); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - } - - @Test - void testHttpsFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { - - final Map settingsMap = new HashMap<>(); - settingsMap.put("request_timeout", 5); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", false); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - pluginSetting = new PluginSetting("otel_trace", settingsMap); - pluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelTraceSourceConfig.class); - SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - - SOURCE.start(buffer); - - WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTPS) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTPS) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - } - - @Test - void testHttpFullBytesWithNonUnframedRequests() { - configureObjectUnderTest(); - SOURCE.start(buffer); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.PROTOBUF) - .build(), - HttpData.copyOf(SUCCESS_REQUEST.toByteArray())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.PROTOBUF) - .build(), - HttpData.copyOf(FAILURE_REQUEST.toByteArray())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - } - - @Test - void testHttpFullJsonWithUnframedRequests() throws InvalidProtocolBufferException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - configureObjectUnderTest(); - SOURCE.start(buffer); - - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @Test - void testHttpCompressionWithUnframedRequests() throws IOException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.GZIP); - configureObjectUnderTest(); - SOURCE.start(buffer); - - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") - .build(), - createGZipCompressedPayload(JsonFormat.printer().print(createExportTraceRequest()))) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @Test - void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolBufferException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); - configureObjectUnderTest(); - SOURCE.start(buffer); - - final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path(transformedPath) - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @Test - void testHttpFullJsonWithCustomPathAndAuthHeader_with_successful_response() throws InvalidProtocolBufferException { - when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); - when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); - final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); - - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) - .thenReturn(grpcAuthenticationProvider); - when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", - Map.of( - "username", USERNAME, - "password", PASSWORD - ))); - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); - - configureObjectUnderTest(); - SOURCE.start(buffer); - - final String encodeToString = Base64.getEncoder() - .encodeToString(String.format("%s:%s", USERNAME, PASSWORD).getBytes(StandardCharsets.UTF_8)); - - final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; - - WebClient.of().prepare() - .post("http://127.0.0.1:21890" + transformedPath) - .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportTraceRequest()).getBytes()) - .header("Authorization", "Basic " + encodeToString) - .execute() - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @Test - void testHttpFullJsonWithCustomPathAndAuthHeader_with_unsuccessful_response() throws InvalidProtocolBufferException { - when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); - when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); - final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); - - when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) - .thenReturn(grpcAuthenticationProvider); - when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", - Map.of( - "username", USERNAME, - "password", PASSWORD - ))); - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); - - configureObjectUnderTest(); - SOURCE.start(buffer); - - final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; - - WebClient.of().prepare() - .post("http://127.0.0.1:21890" + transformedPath) - .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportTraceRequest()).getBytes()) - .execute() - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) - .join(); - } - - @Test - void testServerStartCertFileSuccess() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(server.stop()).thenReturn(completableFuture); - - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", false); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - source.start(buffer); - source.stop(); - - final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); - final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); - verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); - final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); - final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); - assertThat(actualCertificate, is(certAsString)); - assertThat(actualPrivateKey, is(keyAsString)); - } - } - - @Test - void testServerStartACMCertSuccess() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - - final ArgumentCaptor certificateIs = ArgumentCaptor.forClass(InputStream.class); - final ArgumentCaptor privateKeyIs = ArgumentCaptor.forClass(InputStream.class); - verify(serverBuilder).tls(certificateIs.capture(), privateKeyIs.capture()); - final String actualCertificate = IOUtils.toString(certificateIs.getValue(), StandardCharsets.UTF_8.name()); - final String actualPrivateKey = IOUtils.toString(privateKeyIs.getValue(), StandardCharsets.UTF_8.name()); - assertThat(actualCertificate, is(certAsString)); - assertThat(actualPrivateKey, is(keyAsString)); - } - } - - @Test - void start_with_Health_configured_includes_HealthCheck_service() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); - MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - settingsMap.put("health_check_service", "true"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - } - - verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); - verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); - verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); - verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); - } - - @Test - void start_with_Health_configured_unframed_requests_includes_HTTPHealthCheck_service() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); - MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - settingsMap.put("health_check_service", "true"); - settingsMap.put("unframed_requests", "true"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - } - - verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); - verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); - verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); - verify(serverBuilder).service(eq("/health"), isA(HealthCheckService.class)); - } - - @Test - void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); - MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - settingsMap.put("health_check_service", "false"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - } - - verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); - verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); - verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); - verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); - } - - // todo tlongo remove everything related to unframed requests? - @Test - void start_without_Health_configured_unframed_requests_does_not_include_HealthCheck_service() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); - MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - settingsMap.put("health_check_service", "false"); - settingsMap.put("unframed_requests", "true"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - } - - verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); - verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); - verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); - verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); - } - - @Test - void testHealthCheckUnauthNotAllowed() { - // Prepare - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, false); - settingsMap.put("health_check_service", "true"); - settingsMap.put("unframed_requests", "true"); - settingsMap.put("proto_reflection_service", "true"); - settingsMap.put("unauthenticated_health_check", "false"); - settingsMap.put("authentication", new PluginModel("http_basic", - Map.of( - "username", "test", - "password", "test2" - ))); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - - source.start(buffer); - - // When - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("localhost:21890") - .method(HttpMethod.GET) - .path("/health") - .build()) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) - .join(); - - source.stop(); - } - - @Test - void testHealthCheckUnauthAllowed() { - // Prepare - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, false); - settingsMap.put("health_check_service", "true"); - settingsMap.put("unframed_requests", "true"); - settingsMap.put("proto_reflection_service", "true"); - settingsMap.put("unauthenticated_health_check", "true"); - settingsMap.put("authentication", new PluginModel("http_basic", - Map.of( - "username", "test", - "password", "test2" - ))); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - - source.start(buffer); - - // When - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("localhost:21890") - .method(HttpMethod.GET) - .path("/health") - .build()) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)).join(); - - source.stop(); - } - - @Test - void testOptionalHttpAuthServiceNotInPlace() { - when(server.stop()).thenReturn(completableFuture); - - try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - SOURCE.start(buffer); - } - - verify(serverBuilder).service(isA(GrpcService.class)); - verify(serverBuilder, never()).decorator(isA(Function.class)); - } - - @Test - void testOptionalHttpAuthServiceInPlace() { - final Optional> function = Optional.of(httpService -> httpService); - - final Map settingsMap = new HashMap<>(); - settingsMap.put("authentication", new PluginModel("test", null)); - settingsMap.put("unauthenticated_health_check", true); - - settingsMap.put(SSL, false); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - - when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); - - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - - try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - } - - verify(serverBuilder).service(isA(GrpcService.class)); - verify(serverBuilder).decorator(isA(String.class), isA(Function.class)); - } - - @Test - void testOptionalHttpAuthServiceInPlaceWithUnauthenticatedDisabled() { - final Optional> function = Optional.of(httpService -> httpService); - - final Map settingsMap = new HashMap<>(); - settingsMap.put("authentication", new PluginModel("test", null)); - settingsMap.put("unauthenticated_health_check", false); - - settingsMap.put(SSL, false); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - - when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); - - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - - try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - } - - verify(serverBuilder).service(isA(GrpcService.class)); - verify(serverBuilder).decorator(isA(Function.class)); - } - - @Test - void testDoubleStart() { - // starting server - SOURCE.start(buffer); - // double start server - assertThrows(IllegalStateException.class, () -> SOURCE.start(buffer)); - } - - @Test - void testRunAnotherSourceWithSamePort() { - // starting server - SOURCE.start(buffer); - - - Map settingsMap = Map.of("retry_info", TEST_RETRY_INFO, SSL, false); - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - //Expect RuntimeException because when port is already in use, BindException is thrown which is not RuntimeException - assertThrows(RuntimeException.class, () -> source.start(buffer)); - } - - @Test - void testStartWithEmptyBuffer() { - testPluginSetting = new PluginSetting(null, Collections.singletonMap(SSL, false)); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - assertThrows(IllegalStateException.class, () -> source.start(null)); - } - - @Test - void testStartWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(completableFuture.get()).thenThrow(new ExecutionException("", null)); - - // When/Then - assertThrows(RuntimeException.class, () -> source.start(buffer)); - } - } - - @Test - void testStartWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - final NullPointerException expCause = new NullPointerException(); - when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); - - // When/Then - final RuntimeException ex = assertThrows(RuntimeException.class, () -> source.start(buffer)); - assertEquals(expCause, ex); - } - } - - @Test - void testStopWithServerExecutionExceptionNoCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - when(server.stop()).thenReturn(completableFuture); - - // When/Then - when(completableFuture.get()).thenThrow(new ExecutionException("", null)); - assertThrows(RuntimeException.class, source::stop); - } - } - - @Test - void testStartWithInterruptedException() throws ExecutionException, InterruptedException { - // Prepare - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - when(completableFuture.get()).thenThrow(new InterruptedException()); - - // When/Then - assertThrows(RuntimeException.class, () -> source.start(buffer)); - assertTrue(Thread.interrupted()); - } - } - - @Test - void testStopWithServerExecutionExceptionWithCause() throws ExecutionException, InterruptedException { - // Prepare - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - when(server.stop()).thenReturn(completableFuture); - final NullPointerException expCause = new NullPointerException(); - when(completableFuture.get()).thenThrow(new ExecutionException("", expCause)); - - // When/Then - final RuntimeException ex = assertThrows(RuntimeException.class, source::stop); - assertEquals(expCause, ex); - } - } - - @Test - void testStopWithInterruptedException() throws ExecutionException, InterruptedException { - // Prepare - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - when(server.stop()).thenReturn(completableFuture); - when(completableFuture.get()).thenThrow(new InterruptedException()); - - // When/Then - assertThrows(RuntimeException.class, source::stop); - assertTrue(Thread.interrupted()); - } - } - - @Test - void request_that_exceeds_maxRequestLength_returns_413() throws InvalidProtocolBufferException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getMaxRequestLength()).thenReturn(ByteCount.ofBytes(4)); - configureObjectUnderTest(); - SOURCE.start(buffer); - - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.REQUEST_ENTITY_TOO_LARGE, throwable)) - .join(); - } - - @Test - void testServerConnectionsMetric() throws InvalidProtocolBufferException { - // Prepare - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - SOURCE.start(buffer); - - final String metricNamePrefix = new StringJoiner(MetricNames.DELIMITER) - .add("pipeline").add("otel_trace").toString(); - List serverConnectionsMeasurements = MetricsTestUtil.getMeasurementList( - new StringJoiner(MetricNames.DELIMITER).add(metricNamePrefix) - .add(OTelTraceSource.SERVER_CONNECTIONS).toString()); - - // Verify connections metric value is 0 - Measurement serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); - assertEquals(0, serverConnectionsMeasurement.getValue()); - - final RequestHeaders testRequestHeaders = RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(); - final HttpData testHttpData = HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes()); - - // Send request - WebClient.of().execute(testRequestHeaders, testHttpData) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - - // Verify connections metric value is 1 - serverConnectionsMeasurement = MetricsTestUtil.getMeasurementFromList(serverConnectionsMeasurements, Statistic.VALUE); - assertEquals(1.0, serverConnectionsMeasurement.getValue()); - } - private ExportTraceServiceRequest createExportTraceRequest() { - final Span testSpan = Span.newBuilder() + final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) .setName(UUID.randomUUID().toString()) @@ -1078,29 +194,4 @@ private void assertJsonResponse(final String expectedResponseBody, final Aggrega assertThat(body, is(expectedResponseBody)); } - - - private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, - final HttpStatus expectedStatus, - final Throwable throwable) { - assertThat("Http Status", response.status(), equalTo(expectedStatus)); - assertThat("Http Response Throwable", throwable, is(nullValue())); - - final List headerKeys = response.headers() - .stream() - .map(Map.Entry::getKey) - .map(AsciiString::toString) - .collect(Collectors.toList()); - assertThat("Response Header Keys", headerKeys, not(contains("server"))); - } - - private byte[] createGZipCompressedPayload(final String payload) throws IOException { - // Create a GZip compressed request body - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { - gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); - } - return byteStream.toByteArray(); - } - } From 6ebcd570f267c00e5d3bd58e1cbe28299c7d289e Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Wed, 4 Dec 2024 12:22:28 +0100 Subject: [PATCH 05/30] [WIP] Separate concerns when it comes to configuring the server/services Signed-off-by: Tomas Longo --- .../otel-trace-source/build.gradle | 1 + .../source/oteltrace/OTelTraceSource.java | 32 ++++- .../source/oteltrace/grpc/GrpcService.java | 53 +------- .../oteltrace/http/ArmeriaHttpService.java | 12 +- .../oteltrace/http/HttpExceptionHandler.java | 113 ++++++++++++++++++ .../OTelTraceSource_HttpServiceTest.java | 37 +++++- 6 files changed, 185 insertions(+), 63 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index 4a625ee0ff..c84f265d65 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -32,6 +32,7 @@ dependencies { testImplementation 'org.assertj:assertj-core:3.27.0' testImplementation testLibs.slf4j.simple testImplementation libs.commons.io + testImplementation 'com.jayway.jsonpath:json-path-assert:2.6.0' testImplementation 'org.skyscreamer:jsonassert:1.5.3' } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 0460c05cdd..5e82a0301f 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -9,6 +9,7 @@ import com.linecorp.armeria.common.util.BlockingTaskExecutor; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.encoding.DecodingService; import com.linecorp.armeria.server.healthcheck.HealthCheckService; import org.opensearch.dataprepper.metrics.PluginMetrics; @@ -22,11 +23,12 @@ import org.opensearch.dataprepper.model.source.Source; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.plugins.otel.codec.OTelTraceDecoder; import org.opensearch.dataprepper.plugins.source.oteltrace.grpc.GrpcService; -import org.opensearch.dataprepper.plugins.source.oteltrace.http.HttpService; +import org.opensearch.dataprepper.plugins.source.oteltrace.http.ArmeriaHttpService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +44,6 @@ public class OTelTraceSource implements Source> { private static final String HTTP_HEALTH_CHECK_PATH = "/health"; static final String SERVER_CONNECTIONS = "serverConnections"; - private final OTelTraceSourceConfig oTelTraceSourceConfig; private final PluginMetrics pluginMetrics; private final PluginFactory pluginFactory; @@ -88,9 +89,8 @@ public void start(Buffer> buffer) { configureTLS(serverBuilder); configureTaskExecutor(serverBuilder); - // todo tlongo convert to factory method? - new GrpcService(pluginFactory, oTelTraceSourceConfig, pluginMetrics, pipelineName, certificateProviderFactory).create(buffer, serverBuilder); - new HttpService(oTelTraceSourceConfig, pluginMetrics).create(serverBuilder, buffer); + configureGrpcService(serverBuilder, buffer); + configureHttpService(serverBuilder, buffer); server = serverBuilder.build(); @@ -113,7 +113,26 @@ private void handleExecutionException(ExecutionException ex) { } else { throw new RuntimeException(ex); } + } + + private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { + com.linecorp.armeria.server.grpc.GrpcService grpcService = new GrpcService(pluginFactory, oTelTraceSourceConfig, pluginMetrics, pipelineName, certificateProviderFactory).create(buffer, serverBuilder); + + if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { + serverBuilder.service(grpcService); + } else { + serverBuilder.service(grpcService, DecodingService.newDecorator()); + } + } + + private void configureHttpService(ServerBuilder serverBuilder, Buffer> buffer) { + ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics); + if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { + serverBuilder.annotatedService(httpService); + } else { + serverBuilder.annotatedService(httpService, DecodingService.newDecorator()); + } } private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { @@ -129,7 +148,8 @@ private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { } private void configureTLS(ServerBuilder serverBuilder) { - if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { LOG.info("SSL/TLS is enabled."); + if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { + LOG.info("SSL/TLS is enabled."); final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); final Certificate certificate = certificateProvider.getCertificate(); serverBuilder.https(oTelTraceSourceConfig.getPort()).tls( diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index e0ba556b21..a0377cc11e 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -1,7 +1,5 @@ package org.opensearch.dataprepper.plugins.source.oteltrace.grpc; -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.Collections; import java.util.List; @@ -16,9 +14,6 @@ import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; -import org.opensearch.dataprepper.plugins.certificate.model.Certificate; -import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.plugins.health.HealthGrpcService; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; @@ -29,12 +24,9 @@ import org.slf4j.LoggerFactory; import com.linecorp.armeria.common.grpc.GrpcExceptionHandlerFunction; -import com.linecorp.armeria.common.util.BlockingTaskExecutor; import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.encoding.DecodingService; import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; -import com.linecorp.armeria.server.healthcheck.HealthCheckService; import io.grpc.MethodDescriptor; import io.grpc.ServerInterceptor; @@ -68,7 +60,7 @@ public GrpcService(PluginFactory pluginFactory, OTelTraceSourceConfig oTelTraceS this.certificateProviderFactory = certificateProviderFactory; } - public void create(Buffer> buffer, ServerBuilder serverBuilder) { + public com.linecorp.armeria.server.grpc.GrpcService create(Buffer> buffer, ServerBuilder serverBuilder) { final OTelTraceGrpcService oTelTraceGrpcService = new OTelTraceGrpcService( (int)(oTelTraceSourceConfig.getRequestTimeoutInMillis() * 0.8), @@ -95,31 +87,21 @@ public void create(Buffer> buffer, ServerBuilder serverBuilder) { grpcServiceBuilder.addService(ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors)); } + // todo tlongo extract into separate grpc config. Can't we have only one healthcheck for the whole server? We are already configuring one OtelTraceSource if (oTelTraceSourceConfig.hasHealthCheck()) { LOG.info("Health check is enabled"); grpcServiceBuilder.addService(new HealthGrpcService()); } + // todo tlongo extract into separate grpc config if (oTelTraceSourceConfig.hasProtoReflectionService()) { LOG.info("Proto reflection service is enabled"); grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); } - // todo tlongo let this method return the grpc service. All things serverbuilder related have to be done by the source - grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); -// serverBuilder.disableServerHeader(); - if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { - serverBuilder.service(grpcServiceBuilder.build()); - } else { - serverBuilder.service(grpcServiceBuilder.build(), DecodingService.newDecorator()); - } - -// if (oTelTraceSourceConfig.enableHttpHealthCheck()) { -// serverBuilder.service(HTTP_HEALTH_CHECK_PATH, HealthCheckService.builder().longPolling(0).build()); -// } - + // todo tlongo extract to otelTraceSource if (oTelTraceSourceConfig.getAuthentication() != null) { final Optional> optionalHttpAuthenticationService = authenticationProvider.getHttpAuthenticationService(); @@ -132,32 +114,7 @@ public void create(Buffer> buffer, ServerBuilder serverBuilder) { } } -// serverBuilder.requestTimeoutMillis(oTelTraceSourceConfig.getRequestTimeoutInMillis()); -// if(oTelTraceSourceConfig.getMaxRequestLength() != null) { -// serverBuilder.maxRequestLength(oTelTraceSourceConfig.getMaxRequestLength().getBytes()); -// } - - // ACM Cert for SSL takes preference -// if (oTelTraceSourceConfig.isSsl() || oTelTraceSourceConfig.useAcmCertForSSL()) { LOG.info("SSL/TLS is enabled."); -// final CertificateProvider certificateProvider = certificateProviderFactory.getCertificateProvider(); -// final Certificate certificate = certificateProvider.getCertificate(); -// serverBuilder.https(oTelTraceSourceConfig.getPort()).tls( -// new ByteArrayInputStream(certificate.getCertificate().getBytes(StandardCharsets.UTF_8)), -// new ByteArrayInputStream(certificate.getPrivateKey().getBytes(StandardCharsets.UTF_8) -// ) -// ); -// } else { -// LOG.warn("Creating otel_trace_source without SSL/TLS. This is not secure."); -// LOG.warn("In order to set up TLS for the otel_trace_source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#ssl"); -// serverBuilder.http(oTelTraceSourceConfig.getPort()); -// } - -// serverBuilder.maxNumConnections(oTelTraceSourceConfig.getMaxConnectionCount()); -// final BlockingTaskExecutor blockingTaskExecutor = BlockingTaskExecutor.builder() -// .numThreads(oTelTraceSourceConfig.getThreadCount()) -// .threadNamePrefix(pipelineName + "-otel_trace") -// .build(); -// serverBuilder.blockingTaskExecutor(blockingTaskExecutor, true); + return grpcServiceBuilder.build(); } private List getAuthenticationInterceptor() { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index aa7dbe20ce..13f9fc15c3 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -14,21 +14,21 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.trace.Span; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; -import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.armeria.server.annotation.Consumes; +import com.linecorp.armeria.server.annotation.ExceptionHandler; import com.linecorp.armeria.server.annotation.Post; -import io.grpc.stub.StreamObserver; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Timer; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; +@ExceptionHandler(HttpExceptionHandler.class) public class ArmeriaHttpService { private static final Logger LOG = LoggerFactory.getLogger(ArmeriaHttpService.class); @@ -62,17 +62,19 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu requestProcessDuration = pluginMetrics.timer(REQUEST_PROCESS_DURATION); } - // todo tlongo handle excpetions + // todo tlongo handle excpetions --> https://armeria.dev/docs/server-annotated-service#handling-exceptions // todo tlongo handle backoff // todo tlongo healthcheck? - @Post("/hello") + @Post("/opentelemetry.proto.collector.trace.v1.TraceService/Export") @Consumes(value = "application/json") - public void hello(ExportTraceServiceRequest request) { + public ExportTraceServiceResponse exportTrace(ExportTraceServiceRequest request) { requestsReceivedCounter.increment(); payloadSizeSummary.record(request.getSerializedSize()); requestProcessDuration.record(() -> processRequest(request)); + + return ExportTraceServiceResponse.newBuilder().build(); } // todo tlongo exract in order to be used by http and grpc? diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java new file mode 100644 index 0000000000..861db58176 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java @@ -0,0 +1,113 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace.http; + + +import java.util.concurrent.TimeoutException; + +import org.opensearch.dataprepper.exceptions.BadRequestException; +import org.opensearch.dataprepper.exceptions.BufferWriteException; +import org.opensearch.dataprepper.exceptions.RequestCancelledException; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.protobuf.Any; +import com.google.protobuf.Duration; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.google.rpc.RetryInfo; +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.server.RequestTimeoutException; +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.annotation.ExceptionHandlerFunction; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +// todo tlongo add metrics. See GrpcExceptionHandler +public class HttpExceptionHandler implements ExceptionHandlerFunction { + private static final Logger LOG = LoggerFactory.getLogger(HttpExceptionHandler.class); + + static final String ARMERIA_REQUEST_TIMEOUT_MESSAGE = "Timeout waiting for request to be served. This is usually due to the buffer being full."; + + @Override + public HttpResponse handleException(ServiceRequestContext ctx, + HttpRequest req, Throwable e) { + final Throwable exceptionCause = e instanceof BufferWriteException ? e.getCause() : e; + StatusHolder statusHolder = createStatus(exceptionCause); + + try { + // todo tlongo why do we need this in the first place? + JsonFormat.TypeRegistry typeRegistry = JsonFormat.TypeRegistry.newBuilder() + .add(RetryInfo.getDescriptor()) + .build(); + + JsonFormat.Printer printer = JsonFormat.printer().usingTypeRegistry(typeRegistry); + return HttpResponse.of(statusHolder.getHttpStatus(), MediaType.JSON, printer.print(statusHolder.getStatus())); + } catch (InvalidProtocolBufferException ipbe) { + throw new RuntimeException(ipbe); + } + } + + private StatusHolder createStatus(Throwable e) { + if (e instanceof RequestTimeoutException || e instanceof TimeoutException) { + return new StatusHolder(createStatus(e, Status.Code.RESOURCE_EXHAUSTED), createHttpStatusFromProtoBufStatus(Status.Code.RESOURCE_EXHAUSTED)); + } else if (e instanceof SizeOverflowException) { + return new StatusHolder(createStatus(e, Status.Code.RESOURCE_EXHAUSTED), createHttpStatusFromProtoBufStatus(Status.Code.RESOURCE_EXHAUSTED)); + } else if (e instanceof BadRequestException) { + return new StatusHolder(createStatus(e, Status.Code.INVALID_ARGUMENT), createHttpStatusFromProtoBufStatus(Status.Code.INVALID_ARGUMENT)); + } else if ((e instanceof StatusRuntimeException) && (e.getMessage().contains("Invalid protobuf byte sequence") || e.getMessage().contains("Can't decode compressed frame"))) { + return new StatusHolder(createStatus(e, Status.Code.INVALID_ARGUMENT), createHttpStatusFromProtoBufStatus(Status.Code.INVALID_ARGUMENT)); + } else if (e instanceof RequestCancelledException) { + return new StatusHolder(createStatus(e, Status.Code.CANCELLED), createHttpStatusFromProtoBufStatus(Status.Code.CANCELLED)); + } else { + LOG.error("Unexpected exception handling http request", e); + return new StatusHolder(createStatus(e, Status.Code.INTERNAL), createHttpStatusFromProtoBufStatus(Status.Code.INTERNAL)); + } + } + + private HttpStatus createHttpStatusFromProtoBufStatus(Status.Code status) { + if (status == Status.Code.RESOURCE_EXHAUSTED) { + return HttpStatus.INSUFFICIENT_STORAGE; + } else if (status == Status.Code.INVALID_ARGUMENT) { + return HttpStatus.BAD_REQUEST; + } else { + return HttpStatus.INTERNAL_SERVER_ERROR; + } + } + + private com.google.rpc.Status createStatus(final Throwable e, final Status.Code code) { + com.google.rpc.Status.Builder builder = com.google.rpc.Status.newBuilder().setCode(code.value()); + if (e instanceof RequestTimeoutException) { + builder.setMessage(ARMERIA_REQUEST_TIMEOUT_MESSAGE); + } else { + builder.setMessage(e.getMessage() == null ? code.name() :e.getMessage()); + } + if (code == Status.Code.RESOURCE_EXHAUSTED) { + // todo tlongo use retry calculator + builder.addDetails(Any.pack(RetryInfo.newBuilder().setRetryDelay(Duration.newBuilder().setSeconds(1).setNanos(1000000).build()).build())); + } + return builder.build(); + } + + private static class StatusHolder { + private final HttpStatus httpStatus; + private final com.google.rpc.Status status; + + public StatusHolder(com.google.rpc.Status status, HttpStatus httpStatus) { + this.httpStatus = httpStatus; + this.status = status; + } + + public HttpStatus getHttpStatus() { + return httpStatus; + } + + public com.google.rpc.Status getStatus() { + return status; + } + } + +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index cd659fe539..4fd981dfd6 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -33,12 +33,14 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.metrics.MetricsTestUtil; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; @@ -52,6 +54,7 @@ import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpData; import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; @@ -65,6 +68,7 @@ import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; +import static com.jayway.jsonpath.matchers.JsonPathMatchers.*; @ExtendWith(MockitoExtension.class) class OTelTraceSource_HttpServiceTest { @@ -152,6 +156,8 @@ private void configureObjectUnderTest() { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } + // todo tlongo add test for invalid payload + @Test void testHttpService() throws Exception { when(buffer.isByteBuffer()).thenReturn(true); @@ -163,16 +169,38 @@ void testHttpService() throws Exception { .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") .method(HttpMethod.POST) - .path("/hello") + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") .contentType(MediaType.JSON_UTF_8) .build(), HttpData.copyOf(JsonFormat.printer().print(request).getBytes())) .aggregate() - .whenComplete((response, throwable) -> assertJsonResponse("", response)) + .whenComplete((response, throwable) -> assertThat(response.status(), is(HttpStatus.OK))) .join(); verify(buffer, times(1)).writeBytes(bytesCaptor.capture(), anyString(), anyInt()); } + @Test + void request_that_causes_overflow_exception_should_not_be_written_to_buffer() throws Exception { + Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); + configureObjectUnderTest(); + SOURCE.start(buffer); + ExportTraceServiceRequest request = createExportTraceRequest(); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), HttpData.copyOf(JsonFormat.printer().print(request).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> { + assertThat(response.status(), is(HttpStatus.INSUFFICIENT_STORAGE)); + assertResponseBodyForRetryInformation(response); + }) + .join(); + } + private ExportTraceServiceRequest createExportTraceRequest() { final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) @@ -189,9 +217,10 @@ private ExportTraceServiceRequest createExportTraceRequest() { .build(); } - private void assertJsonResponse(final String expectedResponseBody, final AggregatedHttpResponse response) { + private void assertResponseBodyForRetryInformation(final AggregatedHttpResponse response) { String body = response.content(StandardCharsets.UTF_8); - assertThat(body, is(expectedResponseBody)); + // todo tlongo assert delay value + assertThat(body, hasJsonPath("$.details[0].retryDelay")); } } From 4591822844e08509e1acf124eaaf683446a974ca Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Wed, 4 Dec 2024 14:25:36 +0100 Subject: [PATCH 06/30] [WIP] Use retry calculator to provide backoff info Signed-off-by: Tomas Longo --- .../dataprepper/GrpcRetryInfoCalculator.java | 6 ++--- .../source/oteltrace/OTelTraceSource.java | 8 +++--- .../oteltrace/http/ArmeriaHttpService.java | 15 +++++------ .../oteltrace/http/HttpExceptionHandler.java | 12 ++++++--- .../source/oteltrace/http/HttpService.java | 27 ------------------- 5 files changed, 23 insertions(+), 45 deletions(-) delete mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java index 2b74b0b4bc..1cde444734 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java @@ -6,7 +6,7 @@ import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; -class GrpcRetryInfoCalculator { +public class GrpcRetryInfoCalculator { private final Duration minimumDelay; private final Duration maximumDelay; @@ -14,7 +14,7 @@ class GrpcRetryInfoCalculator { private final AtomicReference lastTimeCalled; private final AtomicReference nextDelay; - GrpcRetryInfoCalculator(Duration minimumDelay, Duration maximumDelay) { + public GrpcRetryInfoCalculator(Duration minimumDelay, Duration maximumDelay) { this.minimumDelay = minimumDelay; this.maximumDelay = maximumDelay; // Create a cushion so that the calculator treats a first quick exception (after prepper startup) as normal request (e.g. does not calculate a backoff) @@ -34,7 +34,7 @@ private static com.google.protobuf.Duration.Builder mapDuration(Duration duratio return com.google.protobuf.Duration.newBuilder().setSeconds(duration.getSeconds()).setNanos(duration.getNano()); } - RetryInfo createRetryInfo() { + public RetryInfo createRetryInfo() { Instant now = Instant.now(); // Is the last time we got called longer ago than the next delay? if (lastTimeCalled.getAndSet(now).isBefore(now.minus(nextDelay.get()))) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 5e82a0301f..c8b90d5ce2 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -29,6 +29,7 @@ import org.opensearch.dataprepper.plugins.otel.codec.OTelTraceDecoder; import org.opensearch.dataprepper.plugins.source.oteltrace.grpc.GrpcService; import org.opensearch.dataprepper.plugins.source.oteltrace.http.ArmeriaHttpService; +import org.opensearch.dataprepper.plugins.source.oteltrace.http.HttpExceptionHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,12 +127,13 @@ private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { - ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics); + ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics, oTelTraceSourceConfig.getRequestTimeoutInMillis()); + HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(oTelTraceSourceConfig.getRetryInfo().getMinDelay(), oTelTraceSourceConfig.getRetryInfo().getMaxDelay()); if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { - serverBuilder.annotatedService(httpService); + serverBuilder.annotatedService(httpService, httpExceptionHandler); } else { - serverBuilder.annotatedService(httpService, DecodingService.newDecorator()); + serverBuilder.annotatedService(httpService, DecodingService.newDecorator(), httpExceptionHandler); } } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index 13f9fc15c3..6c73d1c2fc 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -19,7 +19,6 @@ import com.linecorp.armeria.server.ServiceRequestContext; import com.linecorp.armeria.server.annotation.Consumes; -import com.linecorp.armeria.server.annotation.ExceptionHandler; import com.linecorp.armeria.server.annotation.Post; import io.micrometer.core.instrument.Counter; @@ -28,7 +27,6 @@ import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; -@ExceptionHandler(HttpExceptionHandler.class) public class ArmeriaHttpService { private static final Logger LOG = LoggerFactory.getLogger(ArmeriaHttpService.class); @@ -41,19 +39,20 @@ public class ArmeriaHttpService { public static final String PAYLOAD_SIZE = "payloadSize"; public static final String REQUEST_PROCESS_DURATION = "requestProcessDuration"; - final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder; - final Buffer> buffer; - // todo tlongo config - private int bufferWriteTimeoutInMillis = 10_000; + private final OTelProtoCodec.OTelProtoDecoder oTelProtoDecoder; + private final Buffer> buffer; + + private final int bufferWriteTimeoutInMillis; private final Counter requestsReceivedCounter; private final Counter successRequestsCounter; private final DistributionSummary payloadSizeSummary; private final Timer requestProcessDuration; - public ArmeriaHttpService(Buffer> buffer, final PluginMetrics pluginMetrics) { + public ArmeriaHttpService(Buffer> buffer, final PluginMetrics pluginMetrics, final int bufferWriteTimeoutInMillis) { this.buffer = buffer; this.oTelProtoDecoder = new OTelProtoCodec.OTelProtoDecoder(); + this.bufferWriteTimeoutInMillis = bufferWriteTimeoutInMillis; // todo tlongo encapsulate into own class, since both, grpc and http, should contribute to those requestsReceivedCounter = pluginMetrics.counter(REQUESTS_RECEIVED); @@ -62,8 +61,6 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu requestProcessDuration = pluginMetrics.timer(REQUEST_PROCESS_DURATION); } - // todo tlongo handle excpetions --> https://armeria.dev/docs/server-annotated-service#handling-exceptions - // todo tlongo handle backoff // todo tlongo healthcheck? @Post("/opentelemetry.proto.collector.trace.v1.TraceService/Export") diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java index 861db58176..cd52065a57 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java @@ -1,8 +1,10 @@ package org.opensearch.dataprepper.plugins.source.oteltrace.http; +import java.time.Duration; import java.util.concurrent.TimeoutException; +import org.opensearch.dataprepper.GrpcRetryInfoCalculator; import org.opensearch.dataprepper.exceptions.BadRequestException; import org.opensearch.dataprepper.exceptions.BufferWriteException; import org.opensearch.dataprepper.exceptions.RequestCancelledException; @@ -11,7 +13,6 @@ import org.slf4j.LoggerFactory; import com.google.protobuf.Any; -import com.google.protobuf.Duration; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import com.google.rpc.RetryInfo; @@ -32,6 +33,12 @@ public class HttpExceptionHandler implements ExceptionHandlerFunction { static final String ARMERIA_REQUEST_TIMEOUT_MESSAGE = "Timeout waiting for request to be served. This is usually due to the buffer being full."; + private final GrpcRetryInfoCalculator retryInfoCalculator; + + public HttpExceptionHandler(Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { + this.retryInfoCalculator = new GrpcRetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); + } + @Override public HttpResponse handleException(ServiceRequestContext ctx, HttpRequest req, Throwable e) { @@ -86,8 +93,7 @@ private com.google.rpc.Status createStatus(final Throwable e, final Status.Code builder.setMessage(e.getMessage() == null ? code.name() :e.getMessage()); } if (code == Status.Code.RESOURCE_EXHAUSTED) { - // todo tlongo use retry calculator - builder.addDetails(Any.pack(RetryInfo.newBuilder().setRetryDelay(Duration.newBuilder().setSeconds(1).setNanos(1000000).build()).build())); + builder.addDetails(Any.pack(retryInfoCalculator.createRetryInfo())); } return builder.build(); } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java deleted file mode 100644 index eedc09a73c..0000000000 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.oteltrace.http; - -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.linecorp.armeria.server.ServerBuilder; - -public class HttpService { - private static final Logger LOG = LoggerFactory.getLogger(HttpService.class); - private final OTelTraceSourceConfig oTelTraceSourceConfig; - private final PluginMetrics pluginMetrics; - - public HttpService(OTelTraceSourceConfig oTelTraceSourceConfig, final PluginMetrics pluginMetrics) { - this.oTelTraceSourceConfig = oTelTraceSourceConfig; - this.pluginMetrics = pluginMetrics; - } - - public void create(ServerBuilder serverBuilder, Buffer> buffer) { - // todo tlongo what about tls? - LOG.info("Creating http service"); - serverBuilder.annotatedService(new ArmeriaHttpService(buffer, pluginMetrics)); - } -} From 6957288730870940088e90a6e29fa4ae6537538f Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 13 Dec 2024 09:40:54 +0100 Subject: [PATCH 07/30] [WIP] Add metrics to http exception handler Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 2 +- .../source/oteltrace/grpc/GrpcService.java | 1 - .../oteltrace/http/ArmeriaHttpService.java | 2 +- .../oteltrace/http/HttpExceptionHandler.java | 31 ++++++++++++++++--- .../OTelTraceSource_HttpServiceTest.java | 2 +- 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index c8b90d5ce2..96307b7b6a 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -128,7 +128,7 @@ private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics, oTelTraceSourceConfig.getRequestTimeoutInMillis()); - HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(oTelTraceSourceConfig.getRetryInfo().getMinDelay(), oTelTraceSourceConfig.getRetryInfo().getMaxDelay()); + HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, oTelTraceSourceConfig.getRetryInfo().getMinDelay(), oTelTraceSourceConfig.getRetryInfo().getMaxDelay()); if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { serverBuilder.annotatedService(httpService, httpExceptionHandler); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index a0377cc11e..cebc7ae2bc 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -101,7 +101,6 @@ public com.linecorp.armeria.server.grpc.GrpcService create(Buffer grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); - // todo tlongo extract to otelTraceSource if (oTelTraceSourceConfig.getAuthentication() != null) { final Optional> optionalHttpAuthenticationService = authenticationProvider.getHttpAuthenticationService(); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index 6c73d1c2fc..14a3367d9d 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -54,7 +54,6 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu this.oTelProtoDecoder = new OTelProtoCodec.OTelProtoDecoder(); this.bufferWriteTimeoutInMillis = bufferWriteTimeoutInMillis; - // todo tlongo encapsulate into own class, since both, grpc and http, should contribute to those requestsReceivedCounter = pluginMetrics.counter(REQUESTS_RECEIVED); successRequestsCounter = pluginMetrics.counter(SUCCESS_REQUESTS); payloadSizeSummary = pluginMetrics.summary(PAYLOAD_SIZE); @@ -62,6 +61,7 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu } // todo tlongo healthcheck? + // todo tlongo authentication for http (Auth in Grpc Service is grpc specific) @Post("/opentelemetry.proto.collector.trace.v1.TraceService/Export") @Consumes(value = "application/json") diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java index cd52065a57..bdd69608bd 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java @@ -8,6 +8,7 @@ import org.opensearch.dataprepper.exceptions.BadRequestException; import org.opensearch.dataprepper.exceptions.BufferWriteException; import org.opensearch.dataprepper.exceptions.RequestCancelledException; +import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,22 +27,36 @@ import io.grpc.Status; import io.grpc.StatusRuntimeException; +import io.micrometer.core.instrument.Counter; -// todo tlongo add metrics. See GrpcExceptionHandler +// todo tlongo add tests for metrics public class HttpExceptionHandler implements ExceptionHandlerFunction { private static final Logger LOG = LoggerFactory.getLogger(HttpExceptionHandler.class); static final String ARMERIA_REQUEST_TIMEOUT_MESSAGE = "Timeout waiting for request to be served. This is usually due to the buffer being full."; - + public static final String REQUEST_TIMEOUTS = "requestTimeouts"; + public static final String BAD_REQUESTS = "badRequests"; + public static final String REQUESTS_TOO_LARGE = "requestsTooLarge"; + public static final String INTERNAL_SERVER_ERROR = "internalServerError"; + + private final Counter requestTimeoutsCounter; + private final Counter badRequestsCounter; + private final Counter requestsTooLargeCounter; + private final Counter internalServerErrorCounter; private final GrpcRetryInfoCalculator retryInfoCalculator; - public HttpExceptionHandler(Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { + public HttpExceptionHandler(final PluginMetrics pluginMetrics, Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { + requestTimeoutsCounter = pluginMetrics.counter(REQUEST_TIMEOUTS); + badRequestsCounter = pluginMetrics.counter(BAD_REQUESTS); + requestsTooLargeCounter = pluginMetrics.counter(REQUESTS_TOO_LARGE); + internalServerErrorCounter = pluginMetrics.counter(INTERNAL_SERVER_ERROR); this.retryInfoCalculator = new GrpcRetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); } @Override - public HttpResponse handleException(ServiceRequestContext ctx, - HttpRequest req, Throwable e) { + public HttpResponse handleException(final ServiceRequestContext ctx, + final HttpRequest req, + final Throwable e) { final Throwable exceptionCause = e instanceof BufferWriteException ? e.getCause() : e; StatusHolder statusHolder = createStatus(exceptionCause); @@ -60,17 +75,23 @@ public HttpResponse handleException(ServiceRequestContext ctx, private StatusHolder createStatus(Throwable e) { if (e instanceof RequestTimeoutException || e instanceof TimeoutException) { + requestTimeoutsCounter.increment(); return new StatusHolder(createStatus(e, Status.Code.RESOURCE_EXHAUSTED), createHttpStatusFromProtoBufStatus(Status.Code.RESOURCE_EXHAUSTED)); } else if (e instanceof SizeOverflowException) { + requestsTooLargeCounter.increment(); return new StatusHolder(createStatus(e, Status.Code.RESOURCE_EXHAUSTED), createHttpStatusFromProtoBufStatus(Status.Code.RESOURCE_EXHAUSTED)); } else if (e instanceof BadRequestException) { + badRequestsCounter.increment(); return new StatusHolder(createStatus(e, Status.Code.INVALID_ARGUMENT), createHttpStatusFromProtoBufStatus(Status.Code.INVALID_ARGUMENT)); } else if ((e instanceof StatusRuntimeException) && (e.getMessage().contains("Invalid protobuf byte sequence") || e.getMessage().contains("Can't decode compressed frame"))) { + badRequestsCounter.increment(); return new StatusHolder(createStatus(e, Status.Code.INVALID_ARGUMENT), createHttpStatusFromProtoBufStatus(Status.Code.INVALID_ARGUMENT)); } else if (e instanceof RequestCancelledException) { + requestTimeoutsCounter.increment(); return new StatusHolder(createStatus(e, Status.Code.CANCELLED), createHttpStatusFromProtoBufStatus(Status.Code.CANCELLED)); } else { LOG.error("Unexpected exception handling http request", e); + internalServerErrorCounter.increment(); return new StatusHolder(createStatus(e, Status.Code.INTERNAL), createHttpStatusFromProtoBufStatus(Status.Code.INTERNAL)); } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 4fd981dfd6..696abb1f28 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -180,7 +180,7 @@ void testHttpService() throws Exception { } @Test - void request_that_causes_overflow_exception_should_not_be_written_to_buffer() throws Exception { + void request_that_causes_overflow_exception_should_not_be_written_to_buffer_and_return_retry_information() throws Exception { Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); configureObjectUnderTest(); SOURCE.start(buffer); From c96eee286febf6c9a7c28e0cd335aa67a6d6e108 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 13 Dec 2024 10:56:26 +0100 Subject: [PATCH 08/30] [WIP] Revert accidental changes Signed-off-by: Tomas Longo --- examples/trace-analytics-sample-app/docker-compose.yml | 3 +-- examples/trace_analytics_no_ssl_2x.yml | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/examples/trace-analytics-sample-app/docker-compose.yml b/examples/trace-analytics-sample-app/docker-compose.yml index 7d245f9a1e..d0522ee1ac 100644 --- a/examples/trace-analytics-sample-app/docker-compose.yml +++ b/examples/trace-analytics-sample-app/docker-compose.yml @@ -3,7 +3,7 @@ services: data-prepper: restart: unless-stopped container_name: data-prepper - image: opensearch-data-prepper:2.11.0-SNAPSHOT + image: opensearchproject/data-prepper:2 volumes: - ../trace_analytics_no_ssl_2x.yml:/usr/share/data-prepper/pipelines/pipelines.yaml - ../data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml @@ -21,7 +21,6 @@ services: - discovery.type=single-node - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=M__!!ega123!" ulimits: memlock: soft: -1 diff --git a/examples/trace_analytics_no_ssl_2x.yml b/examples/trace_analytics_no_ssl_2x.yml index 23d7e056bb..a1ce76575a 100644 --- a/examples/trace_analytics_no_ssl_2x.yml +++ b/examples/trace_analytics_no_ssl_2x.yml @@ -16,11 +16,10 @@ raw-pipeline: - otel_traces: sink: - opensearch: - insecure: true hosts: [ "https://node-0.example.com:9200" ] -# cert: "/usr/share/data-prepper/root-ca.pem" + cert: "/usr/share/data-prepper/root-ca.pem" username: "admin" - password: "M__!!ega123!" + password: "admin" index_type: trace-analytics-raw service-map-pipeline: delay: "100" @@ -31,9 +30,8 @@ service-map-pipeline: - service_map: sink: - opensearch: - insecure: true hosts: ["https://node-0.example.com:9200"] -# cert: "/usr/share/data-prepper/root-ca.pem" + cert: "/usr/share/data-prepper/root-ca.pem" username: "admin" - password: "M__!!ega123!" + password: "admin" index_type: trace-analytics-service-map From 2640d44f4c996d84e0501bc66e25e21b2716d161 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 13 Dec 2024 14:19:42 +0100 Subject: [PATCH 09/30] [WIP] Infer protocol from config. Isolate tests regarding unframed requests Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 21 +- .../source/oteltrace/grpc/GrpcService.java | 1 + .../source/oteltrace/OTelTraceSourceTest.java | 292 +---------- .../OTelTraceSource_UnframedRequestsTest.java | 495 ++++++++++++++++++ 4 files changed, 520 insertions(+), 289 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 96307b7b6a..f8f7e1aaa3 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -35,6 +35,7 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.concurrent.ExecutionException; @DataPrepperPlugin(name = "otel_trace_source", pluginType = Source.class, pluginConfigurationType = OTelTraceSourceConfig.class) @@ -42,6 +43,9 @@ public class OTelTraceSource implements Source> { private static final String PLUGIN_NAME = "otel_trace_source"; private static final Logger LOG = LoggerFactory.getLogger(OTelTraceSource.class); + // todo tlongo include in config + private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); + private static final String HTTP_HEALTH_CHECK_PATH = "/health"; static final String SERVER_CONNECTIONS = "serverConnections"; @@ -84,7 +88,7 @@ public void start(Buffer> buffer) { if (server == null) { ServerBuilder serverBuilder = Server.builder(); - serverBuilder = serverBuilder.port(oTelTraceSourceConfig.getPort(), SessionProtocol.HTTP); + serverBuilder = serverBuilder.port(oTelTraceSourceConfig.getPort(), inferProtocolFromConfig()); configureHeadersAndHealthCheck(serverBuilder); configureTLS(serverBuilder); @@ -108,6 +112,14 @@ public void start(Buffer> buffer) { LOG.info("Started otel_trace_source on port " + oTelTraceSourceConfig.getPort() + "..."); } + private SessionProtocol inferProtocolFromConfig() { + if (oTelTraceSourceConfig.isSsl()) { + return SessionProtocol.HTTPS; + } else { + return SessionProtocol.HTTP; + } + } + private void handleExecutionException(ExecutionException ex) { if (ex.getCause() != null && ex.getCause() instanceof RuntimeException) { throw (RuntimeException) ex.getCause(); @@ -128,7 +140,12 @@ private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics, oTelTraceSourceConfig.getRequestTimeoutInMillis()); - HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, oTelTraceSourceConfig.getRetryInfo().getMinDelay(), oTelTraceSourceConfig.getRetryInfo().getMaxDelay()); + RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null + ? oTelTraceSourceConfig.getRetryInfo() + : DEFAULT_RETRY_INFO; + + // todo tlongo move creation of handler to ArmeriaHttpService + HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { serverBuilder.annotatedService(httpService, httpExceptionHandler); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index cebc7ae2bc..7cd6f62605 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -99,6 +99,7 @@ public com.linecorp.armeria.server.grpc.GrpcService create(Buffer grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); } + // todo tlongo still needed with new http-service? grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); if (oTelTraceSourceConfig.getAuthentication() != null) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 40587c398c..959b8ef3f0 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -10,12 +10,11 @@ import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.ClosedSessionException; import com.linecorp.armeria.common.HttpData; -import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpMethod; import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; @@ -30,10 +29,12 @@ import io.grpc.BindableService; import io.grpc.ServerServiceDefinition; import io.grpc.Status; +import io.grpc.StatusRuntimeException; import io.micrometer.core.instrument.Measurement; import io.micrometer.core.instrument.Statistic; import io.netty.util.AsciiString; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc; import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; @@ -42,9 +43,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mock; @@ -58,7 +56,6 @@ import org.opensearch.dataprepper.metrics.MetricsTestUtil; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import org.opensearch.dataprepper.model.configuration.PipelineDescription; import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -73,7 +70,6 @@ import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -91,11 +87,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.zip.GZIPOutputStream; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; @@ -103,11 +96,11 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -125,6 +118,7 @@ @ExtendWith(MockitoExtension.class) class OTelTraceSourceTest { + private static final String GRPC_ENDPOINT = "gproto+http://127.0.0.1:21890/"; private static final String USERNAME = "test_user"; private static final String PASSWORD = "test_password"; private static final String TEST_PATH = "${pipelineName}/v1/traces"; @@ -234,164 +228,6 @@ private void configureObjectUnderTest() { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } - @Test - void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { - configureObjectUnderTest(); - SOURCE.start(buffer); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - } - - @Test - void testHttpsFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { - - final Map settingsMap = new HashMap<>(); - settingsMap.put("request_timeout", 5); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", false); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - pluginSetting = new PluginSetting("otel_trace", settingsMap); - pluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelTraceSourceConfig.class); - SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); - - SOURCE.start(buffer); - - WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTPS) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTPS) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - } - - @Test - void testHttpFullBytesWithNonUnframedRequests() { - configureObjectUnderTest(); - SOURCE.start(buffer); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.PROTOBUF) - .build(), - HttpData.copyOf(SUCCESS_REQUEST.toByteArray())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.PROTOBUF) - .build(), - HttpData.copyOf(FAILURE_REQUEST.toByteArray())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) - .join(); - } - - @Test - void testHttpFullJsonWithUnframedRequests() throws InvalidProtocolBufferException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - configureObjectUnderTest(); - SOURCE.start(buffer); - - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @Test - void testHttpCompressionWithUnframedRequests() throws IOException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.GZIP); - configureObjectUnderTest(); - SOURCE.start(buffer); - - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") - .build(), - createGZipCompressedPayload(JsonFormat.printer().print(createExportTraceRequest()))) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - - @Test - void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolBufferException { - when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); - when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); - configureObjectUnderTest(); - SOURCE.start(buffer); - - final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; - WebClient.of().execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path(transformedPath) - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) - .aggregate() - .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) - .join(); - } - @Test void testHttpFullJsonWithCustomPathAndAuthHeader_with_successful_response() throws InvalidProtocolBufferException { when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); @@ -672,49 +508,6 @@ void start_with_Health_configured_includes_HealthCheck_service() throws IOExcept verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); } - @Test - void start_with_Health_configured_unframed_requests_includes_HTTPHealthCheck_service() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); - MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - settingsMap.put("health_check_service", "true"); - settingsMap.put("unframed_requests", "true"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - } - - verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); - verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); - verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); - verify(serverBuilder).service(eq("/health"), isA(HealthCheckService.class)); - } - @Test void start_without_Health_configured_does_not_include_HealthCheck_service() throws IOException { try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); @@ -756,48 +549,6 @@ void start_without_Health_configured_does_not_include_HealthCheck_service() thro verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); } - @Test - void start_without_Health_configured_unframed_requests_does_not_include_HealthCheck_service() throws IOException { - try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); - MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); - when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); - - when(server.stop()).thenReturn(completableFuture); - final Path certFilePath = Path.of("data/certificate/test_cert.crt"); - final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); - final String certAsString = Files.readString(certFilePath); - final String keyAsString = Files.readString(keyFilePath); - when(certificate.getCertificate()).thenReturn(certAsString); - when(certificate.getPrivateKey()).thenReturn(keyAsString); - when(certificateProvider.getCertificate()).thenReturn(certificate); - when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); - final Map settingsMap = new HashMap<>(); - settingsMap.put(SSL, true); - settingsMap.put("useAcmCertForSSL", true); - settingsMap.put("awsRegion", "us-east-1"); - settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); - settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); - settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); - settingsMap.put("health_check_service", "false"); - settingsMap.put("unframed_requests", "true"); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - source.start(buffer); - source.stop(); - } - - verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); - verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); - verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); - verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); - } - @Test void testHealthCheckUnauthNotAllowed() { // Prepare @@ -1116,29 +867,6 @@ void testServerConnectionsMetric() throws InvalidProtocolBufferException { assertEquals(1.0, serverConnectionsMeasurement.getValue()); } - static class BufferExceptionToStatusArgumentsProvider implements ArgumentsProvider { - @Override - public Stream provideArguments(final ExtensionContext context) { - return Stream.of( - arguments(TimeoutException.class, Status.Code.RESOURCE_EXHAUSTED), - arguments(SizeOverflowException.class, Status.Code.RESOURCE_EXHAUSTED), - arguments(Exception.class, Status.Code.INTERNAL), - arguments(RuntimeException.class, Status.Code.INTERNAL) - ); - } - } - - private ExportTraceServiceRequest createInvalidExportTraceRequest() { - final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() - .setTraceState("SUCCESS").build(); - final ExportTraceServiceRequest successRequest = ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) - .build(); - - return successRequest; - } - private ExportTraceServiceRequest createExportTraceRequest() { final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) @@ -1168,14 +896,4 @@ private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse res .collect(Collectors.toList()); assertThat("Response Header Keys", headerKeys, not(contains("server"))); } - - private byte[] createGZipCompressedPayload(final String payload) throws IOException { - // Create a GZip compressed request body - final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { - gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); - } - return byteStream.toByteArray(); - } - } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java new file mode 100644 index 0000000000..1d19507225 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java @@ -0,0 +1,495 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.oteltrace; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; +import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.metrics.MetricsTestUtil; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; +import org.opensearch.dataprepper.plugins.certificate.model.Certificate; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.health.HealthGrpcService; +import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import com.linecorp.armeria.client.ClientFactory; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpHeaderNames; +import com.linecorp.armeria.common.HttpMethod; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.common.RequestHeaders; +import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; + +import io.grpc.BindableService; +import io.grpc.ServerServiceDefinition; +import io.netty.util.AsciiString; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; +import io.opentelemetry.proto.trace.v1.Span; + + +// todo tlongo check if unframed requests are still needed. If not, remove this whole test class +@ExtendWith(MockitoExtension.class) +class OTelTraceSource_UnframedRequestsTest { + // used to configure the path for unframed requests and make sure not to use the same path + // as the http service + private static final String UNFRAMED_REQUESTS_PATH = "/unframed"; + private static final String TEST_PATH = "${pipelineName}/v1/traces"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); + private static final String TEST_PIPELINE_NAME = "test_pipeline"; + private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); + private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() + .addSpans(Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); + private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder() + .addSpans(Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); + + @Mock + private ServerBuilder serverBuilder; + + @Mock + private Server server; + + @Mock + private GrpcServiceBuilder grpcServiceBuilder; + + @Mock + private GrpcService grpcService; + + @Mock + private CertificateProviderFactory certificateProviderFactory; + + @Mock + private CertificateProvider certificateProvider; + + @Mock + private Certificate certificate; + + @Mock + private CompletableFuture completableFuture; + + @Mock + private PluginFactory pluginFactory; + + @Mock + private GrpcBasicAuthenticationProvider authenticationProvider; + + @Mock(lenient = true) + private OTelTraceSourceConfig oTelTraceSourceConfig; + + @Mock + private Buffer> buffer; + + private PluginSetting pluginSetting; + private PluginSetting testPluginSetting; + private PluginMetrics pluginMetrics; + private PipelineDescription pipelineDescription; + private OTelTraceSource SOURCE; + + @BeforeEach + void beforeEach() { + lenient().when(serverBuilder.port(anyInt(), ArgumentMatchers.any())).thenReturn(serverBuilder); + lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.service(any(GrpcService.class), any(Function.class))).thenReturn(serverBuilder); + lenient().when(serverBuilder.http(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.https(anyInt())).thenReturn(serverBuilder); + lenient().when(serverBuilder.build()).thenReturn(server); + + lenient().when(server.start()).thenReturn(completableFuture); + + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any( + GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); + + lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); + + when(oTelTraceSourceConfig.getPath()).thenReturn(UNFRAMED_REQUESTS_PATH); + when(oTelTraceSourceConfig.getPort()).thenReturn(DEFAULT_PORT); + when(oTelTraceSourceConfig.isSsl()).thenReturn(false); + when(oTelTraceSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); + when(oTelTraceSourceConfig.getMaxConnectionCount()).thenReturn(10); + when(oTelTraceSourceConfig.getThreadCount()).thenReturn(5); + when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); + when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); + + lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(authenticationProvider); + configureObjectUnderTest(); + pipelineDescription = mock(PipelineDescription.class); + lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + } + + @AfterEach + void afterEach() { + SOURCE.stop(); + } + + private void configureObjectUnderTest() { + MetricsTestUtil.initMetrics(); + pluginMetrics = PluginMetrics.fromNames("otel_trace", "pipeline"); + + pipelineDescription = mock(PipelineDescription.class); + when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); + SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + } + + @Test + void testHttpFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpsFullJsonWithNonUnframedRequests() throws InvalidProtocolBufferException { + + final Map settingsMap = new HashMap<>(); + settingsMap.put("request_timeout", 5); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", false); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("path", UNFRAMED_REQUESTS_PATH); + pluginSetting = new PluginSetting("otel_trace", settingsMap); + pluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(pluginSetting.getSettings(), OTelTraceSourceConfig.class); + SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); + + SOURCE.start(buffer); + + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(SUCCESS_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.builder().factory(ClientFactory.insecure()).build().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTPS) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(FAILURE_REQUEST).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullBytesWithNonUnframedRequests() { + configureObjectUnderTest(); + SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(SUCCESS_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.PROTOBUF) + .build(), + HttpData.copyOf(FAILURE_REQUEST.toByteArray())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNSUPPORTED_MEDIA_TYPE, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithUnframedRequests() throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpCompressionWithUnframedRequests() throws IOException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.GZIP); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(UNFRAMED_REQUESTS_PATH) + .contentType(MediaType.JSON_UTF_8) + .add(HttpHeaderNames.CONTENT_ENCODING, "gzip") + .build(), + createGZipCompressedPayload(JsonFormat.printer().print(createExportTraceRequest()))) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + @Test + void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.enableUnframedRequests()).thenReturn(true); + when(oTelTraceSourceConfig.getPath()).thenReturn(TEST_PATH); + configureObjectUnderTest(); + SOURCE.start(buffer); + + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/traces"; + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(transformedPath) + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.OK, throwable)) + .join(); + } + + + @Test + void start_with_Health_configured_unframed_requests_includes_HTTPHealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "true"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder).addService(isA(HealthGrpcService.class)); + verify(serverBuilder).service(eq("/health"), isA(HealthCheckService.class)); + } + + + @Test + void start_without_Health_configured_unframed_requests_does_not_include_HealthCheck_service() throws IOException { + try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class); + MockedStatic grpcServerMock = Mockito.mockStatic(GrpcService.class)) { + armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); + grpcServerMock.when(GrpcService::builder).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.addService(any(ServerServiceDefinition.class))).thenReturn(grpcServiceBuilder); + when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); + + when(server.stop()).thenReturn(completableFuture); + final Path certFilePath = Path.of("data/certificate/test_cert.crt"); + final Path keyFilePath = Path.of("data/certificate/test_decrypted_key.key"); + final String certAsString = Files.readString(certFilePath); + final String keyAsString = Files.readString(keyFilePath); + when(certificate.getCertificate()).thenReturn(certAsString); + when(certificate.getPrivateKey()).thenReturn(keyAsString); + when(certificateProvider.getCertificate()).thenReturn(certificate); + when(certificateProviderFactory.getCertificateProvider()).thenReturn(certificateProvider); + final Map settingsMap = new HashMap<>(); + settingsMap.put(SSL, true); + settingsMap.put("useAcmCertForSSL", true); + settingsMap.put("awsRegion", "us-east-1"); + settingsMap.put("acmCertificateArn", "arn:aws:acm:us-east-1:account:certificate/1234-567-856456"); + settingsMap.put("sslKeyCertChainFile", "data/certificate/test_cert.crt"); + settingsMap.put("sslKeyFile", "data/certificate/test_decrypted_key.key"); + settingsMap.put("health_check_service", "false"); + settingsMap.put("unframed_requests", "true"); + + testPluginSetting = new PluginSetting(null, settingsMap); + testPluginSetting.setPipelineName("pipeline"); + oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); + final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); + source.start(buffer); + source.stop(); + } + + verify(grpcServiceBuilder, times(1)).useClientTimeoutHeader(false); + verify(grpcServiceBuilder, times(1)).useBlockingTaskExecutor(true); + verify(grpcServiceBuilder, never()).addService(isA(HealthGrpcService.class)); + verify(serverBuilder, never()).service(eq("/health"),isA(HealthCheckService.class)); + } + + private ExportTraceServiceRequest createExportTraceRequest() { + final Span testSpan = Span.newBuilder() + .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + } + + private void assertSecureResponseWithStatusCode(final AggregatedHttpResponse response, + final HttpStatus expectedStatus, + final Throwable throwable) { + assertThat("Http Status", response.status(), equalTo(expectedStatus)); + assertThat("Http Response Throwable", throwable, is(nullValue())); + + final List headerKeys = response.headers() + .stream() + .map(Map.Entry::getKey) + .map(AsciiString::toString) + .collect(Collectors.toList()); + assertThat("Response Header Keys", headerKeys, not(contains("server"))); + } + + private byte[] createGZipCompressedPayload(final String payload) throws IOException { + // Create a GZip compressed request body + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (final GZIPOutputStream gzipStream = new GZIPOutputStream(byteStream)) { + gzipStream.write(payload.getBytes(StandardCharsets.UTF_8)); + } + return byteStream.toByteArray(); + } + +} From 2becfde2ddf34745096d59fc750752dd29fee605 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Tue, 17 Dec 2024 16:08:37 +0100 Subject: [PATCH 10/30] [WIP] Add basic auth to http service Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 44 ++++++- .../source/oteltrace/grpc/GrpcService.java | 3 +- .../source/oteltrace/OTelTraceSourceTest.java | 8 -- .../OTelTraceSource_HttpServiceTest.java | 119 +++++++++++++++--- 4 files changed, 143 insertions(+), 31 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index f8f7e1aaa3..8c8659d55b 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -5,22 +5,29 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import static org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME; + import com.linecorp.armeria.common.SessionProtocol; import com.linecorp.armeria.common.util.BlockingTaskExecutor; +import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.encoding.DecodingService; import com.linecorp.armeria.server.healthcheck.HealthCheckService; +import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; +import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; @@ -36,7 +43,10 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; +import java.util.function.Function; @DataPrepperPlugin(name = "otel_trace_source", pluginType = Source.class, pluginConfigurationType = OTelTraceSourceConfig.class) public class OTelTraceSource implements Source> { @@ -87,8 +97,7 @@ public void start(Buffer> buffer) { } if (server == null) { - ServerBuilder serverBuilder = Server.builder(); - serverBuilder = serverBuilder.port(oTelTraceSourceConfig.getPort(), inferProtocolFromConfig()); + ServerBuilder serverBuilder = Server.builder().port(oTelTraceSourceConfig.getPort(), inferProtocolFromConfig()); configureHeadersAndHealthCheck(serverBuilder); configureTLS(serverBuilder); @@ -147,6 +156,8 @@ private void configureHttpService(ServerBuilder serverBuilder, Buffer Create additional layer. See GrpcService + private void configureAuthentication(ServerBuilder serverBuilder) { + if (oTelTraceSourceConfig.getAuthentication() == null || oTelTraceSourceConfig.getAuthentication().getPluginName().equals(UNAUTHENTICATED_PLUGIN_NAME)) { + LOG.warn("Creating otel_trace_source http service without authentication. This is not secure."); + LOG.warn("In order to set up Http Basic authentication for the otel-trace-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#authentication-configurations"); + } else { + ArmeriaHttpAuthenticationProvider authenticationProvider = createAuthenticationProvider(oTelTraceSourceConfig.getAuthentication()); + authenticationProvider.getAuthenticationDecorator().ifPresent(serverBuilder::decorator); + } + } + + // todo tlongo move to http service -> Create additional layer. See GrpcService + private ArmeriaHttpAuthenticationProvider createAuthenticationProvider(final PluginModel authenticationConfiguration) { + Map pluginSettings = authenticationConfiguration.getPluginSettings(); + + // controversial + // the world would be a nicer place, if mere configs were not be treated as plugins + // this method replaces the process of + // yaml -> pluginmodel -> pluginsettings -> configPojo -> pluginfactory -> provider + // with + // yaml -> configPojo -> provider (we could eliminate using Plugin* Classes all together by parsing the yaml section at startup, e.g. like retryInfo) + // pros: + // - we can easily reason about the origins of the provider + // - it becomes testable + // cons: + // - currently tied to one impl by using 'new'. + return new HttpBasicArmeriaHttpAuthenticationProvider(new HttpBasicAuthenticationConfig(pluginSettings.get("username").toString(), pluginSettings.get("password").toString())); + } + private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { serverBuilder.disableServerHeader(); if (oTelTraceSourceConfig.enableHttpHealthCheck()) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index 7cd6f62605..1eff3287b2 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -50,6 +50,7 @@ public class GrpcService { private final GrpcAuthenticationProvider authenticationProvider; private final PluginMetrics pluginMetrics; private final String pipelineName; + // todo tlongo check why unused private final CertificateProviderFactory certificateProviderFactory; public GrpcService(PluginFactory pluginFactory, OTelTraceSourceConfig oTelTraceSourceConfig, PluginMetrics pluginMetrics, String pipelineName, CertificateProviderFactory certificateProviderFactory) { @@ -129,7 +130,7 @@ private GrpcAuthenticationProvider createAuthenticationProvider(final PluginFact final PluginModel authenticationConfiguration = oTelTraceSourceConfig.getAuthentication(); if (authenticationConfiguration == null || authenticationConfiguration.getPluginName().equals(GrpcAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME)) { - LOG.warn("Creating otel-trace-source without authentication. This is not secure."); + LOG.warn("Creating otel_trace_source grpc service without authentication. This is not secure."); LOG.warn("In order to set up Http Basic authentication for the otel-trace-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#authentication-configurations"); } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 959b8ef3f0..23e6c39db6 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -125,14 +125,6 @@ class OTelTraceSourceTest { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().registerModule(new JavaTimeModule()); private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); - private static final ExportTraceServiceRequest SUCCESS_REQUEST = ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .addScopeSpans(ScopeSpans.newBuilder() - .addSpans(io.opentelemetry.proto.trace.v1.Span.newBuilder().setTraceState("SUCCESS").build())).build()).build(); - private static final ExportTraceServiceRequest FAILURE_REQUEST = ExportTraceServiceRequest.newBuilder() - .addResourceSpans(ResourceSpans.newBuilder() - .addScopeSpans(ScopeSpans.newBuilder() - .addSpans(io.opentelemetry.proto.trace.v1.Span.newBuilder().setTraceState("FAILURE").build())).build()).build(); @Mock private ServerBuilder serverBuilder; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 696abb1f28..82d398df1a 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -7,6 +7,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Named.named; +import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -22,33 +24,42 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.MetricsTestUtil; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.buffer.SizeOverflowException; import org.opensearch.dataprepper.model.configuration.PipelineDescription; +import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; import org.opensearch.dataprepper.plugins.codec.CompressionOption; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; @@ -58,6 +69,7 @@ import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; +import com.linecorp.armeria.internal.shaded.bouncycastle.util.encoders.Base64; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -93,15 +105,15 @@ class OTelTraceSource_HttpServiceTest { @Mock private PluginFactory pluginFactory; - @Mock - private GrpcBasicAuthenticationProvider authenticationProvider; - @Mock(lenient = true) private OTelTraceSourceConfig oTelTraceSourceConfig; @Mock private Buffer> buffer; + @Mock + private GrpcAuthenticationProvider grpcAuthProvider; + @Captor ArgumentCaptor bytesCaptor; @@ -109,6 +121,9 @@ class OTelTraceSource_HttpServiceTest { private PipelineDescription pipelineDescription; private OTelTraceSource SOURCE; + private static HttpBasicAuthenticationConfig PROVIDED_CONFIG = new HttpBasicAuthenticationConfig("username", "password"); + + @BeforeEach void beforeEach() { lenient().when(serverBuilder.service(any(GrpcService.class))).thenReturn(serverBuilder); @@ -118,15 +133,14 @@ void beforeEach() { lenient().when(serverBuilder.build()).thenReturn(server); lenient().when(server.start()).thenReturn(completableFuture); + lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(grpcAuthProvider); + lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.exceptionHandler(any( - GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any(GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); - lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); - when(oTelTraceSourceConfig.getPort()).thenReturn(DEFAULT_PORT); when(oTelTraceSourceConfig.isSsl()).thenReturn(false); when(oTelTraceSourceConfig.getRequestTimeoutInMillis()).thenReturn(DEFAULT_REQUEST_TIMEOUT_MS); @@ -135,8 +149,9 @@ void beforeEach() { when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); - lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) - .thenReturn(authenticationProvider); + // default: we don't want authentication + when(oTelTraceSourceConfig.getAuthentication()).thenReturn(null); + configureObjectUnderTest(); pipelineDescription = mock(PipelineDescription.class); lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); @@ -159,12 +174,11 @@ private void configureObjectUnderTest() { // todo tlongo add test for invalid payload @Test - void testHttpService() throws Exception { + void request_that_is_successful() throws Exception { when(buffer.isByteBuffer()).thenReturn(true); ExportTraceServiceRequest request = createExportTraceRequest(); - - configureObjectUnderTest(); SOURCE.start(buffer); + WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") @@ -179,28 +193,93 @@ void testHttpService() throws Exception { verify(buffer, times(1)).writeBytes(bytesCaptor.capture(), anyString(), anyInt()); } + @Test + void providing_unauthenticated_via_config_does_not_add_the_auth_decorator() { + when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel(ArmeriaHttpAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME, Map.of())); + SOURCE.start(buffer); + + verify(serverBuilder, times(0)).decorator(any(Function.class)); + } + @Test void request_that_causes_overflow_exception_should_not_be_written_to_buffer_and_return_retry_information() throws Exception { Mockito.lenient().doThrow(SizeOverflowException.class).when(buffer).writeAll(any(), anyInt()); - configureObjectUnderTest(); SOURCE.start(buffer); - ExportTraceServiceRequest request = createExportTraceRequest(); + + makeRequestAndAssertResponse("/opentelemetry.proto.collector.trace.v1.TraceService/Export", createExportTraceRequest(), (response, throwable) -> { + assertThat(response.status(), is(HttpStatus.INSUFFICIENT_STORAGE)); + assertResponseBodyForRetryInformation(response); + }); + } + + @ParameterizedTest + @MethodSource("generateCredentials") + void request_with_credentials_returns_expected_status_code(AuthTestDataHolder testData) throws InvalidProtocolBufferException { + when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", Map.of("username", PROVIDED_CONFIG.getUsername(), "password",PROVIDED_CONFIG.getPassword()))); + SOURCE.start(buffer); + + makeRequestWithCredentialsAndAssertResponse("/opentelemetry.proto.collector.trace.v1.TraceService/Export", + createExportTraceRequest(), + testData.providedCredentials.getOrDefault("username", null), + testData.providedCredentials.getOrDefault("password", null), + (response, throwable) -> assertThat(response.status(), is(testData.expectedStatus)) + ); + } + + private static Stream generateCredentials() { + return Stream.of( + arguments(named("valid credentials", new AuthTestDataHolder(Map.of("username", "username", "password","password"), HttpStatus.OK))), + arguments(named("wrong credentials", new AuthTestDataHolder(Map.of("username", "wrong-username", "password","wrong-password"), HttpStatus.UNAUTHORIZED))), + arguments(named("no credentials provided", new AuthTestDataHolder(Map.of(), HttpStatus.UNAUTHORIZED))) + ); + } + + static class AuthTestDataHolder { + Map providedCredentials; + HttpStatus expectedStatus; + + public AuthTestDataHolder(Map providedCredentials, HttpStatus expectedStatus) { + this.providedCredentials = providedCredentials; + this.expectedStatus = expectedStatus; + } + } + + void makeRequestWithCredentialsAndAssertResponse( + String path, + ExportTraceServiceRequest request, + String username, + String password, + BiConsumer assertionFunction) throws InvalidProtocolBufferException { + + WebClient.of().execute(RequestHeaders.builder().add("Authorization", "Basic " + new String(Base64.encode(String.format("%s:%s", username, password).getBytes()))) + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path(path) + .contentType(MediaType.JSON_UTF_8) + .build(), HttpData.copyOf(JsonFormat.printer().print(request).getBytes())) + .aggregate() + .whenComplete(assertionFunction) + .join(); + } + + private void makeRequestAndAssertResponse(String path, ExportTraceServiceRequest request, BiConsumer assertionFunction) throws InvalidProtocolBufferException { WebClient.of().execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21890") .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .path(path) .contentType(MediaType.JSON_UTF_8) .build(), HttpData.copyOf(JsonFormat.printer().print(request).getBytes())) .aggregate() - .whenComplete((response, throwable) -> { - assertThat(response.status(), is(HttpStatus.INSUFFICIENT_STORAGE)); - assertResponseBodyForRetryInformation(response); - }) + .whenComplete(assertionFunction) .join(); } + + // todo tlongo https test + private ExportTraceServiceRequest createExportTraceRequest() { final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) From b084fa79547d65d519a77e4d0afa2b592c5ce4e0 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 20 Dec 2024 11:31:47 +0100 Subject: [PATCH 11/30] [WIP] Move configuration of http service into own class Signed-off-by: Tomas Longo --- .../dataprepper/GrpcRetryInfoCalculator.java | 1 + .../source/oteltrace/OTelTraceSource.java | 63 +-------------- .../oteltrace/http/ArmeriaHttpService.java | 2 +- .../source/oteltrace/http/HttpService.java | 81 +++++++++++++++++++ .../source/oteltrace/OTelTraceSourceTest.java | 26 +----- .../OTelTraceSource_HttpServiceTest.java | 44 +++++++--- 6 files changed, 121 insertions(+), 96 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java index 1cde444734..6c7e3d8bf0 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java @@ -6,6 +6,7 @@ import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; +// todo tlongo rename public class GrpcRetryInfoCalculator { private final Duration minimumDelay; diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 8c8659d55b..26647b30bd 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -5,29 +5,22 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; -import static org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME; - import com.linecorp.armeria.common.SessionProtocol; import com.linecorp.armeria.common.util.BlockingTaskExecutor; -import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.encoding.DecodingService; import com.linecorp.armeria.server.healthcheck.HealthCheckService; -import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.model.configuration.PipelineDescription; -import org.opensearch.dataprepper.model.configuration.PluginModel; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; -import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; @@ -35,26 +28,19 @@ import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.plugins.otel.codec.OTelTraceDecoder; import org.opensearch.dataprepper.plugins.source.oteltrace.grpc.GrpcService; -import org.opensearch.dataprepper.plugins.source.oteltrace.http.ArmeriaHttpService; -import org.opensearch.dataprepper.plugins.source.oteltrace.http.HttpExceptionHandler; +import org.opensearch.dataprepper.plugins.source.oteltrace.http.HttpService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Map; -import java.util.Optional; import java.util.concurrent.ExecutionException; -import java.util.function.Function; @DataPrepperPlugin(name = "otel_trace_source", pluginType = Source.class, pluginConfigurationType = OTelTraceSourceConfig.class) public class OTelTraceSource implements Source> { private static final String PLUGIN_NAME = "otel_trace_source"; private static final Logger LOG = LoggerFactory.getLogger(OTelTraceSource.class); - // todo tlongo include in config - private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); private static final String HTTP_HEALTH_CHECK_PATH = "/health"; static final String SERVER_CONNECTIONS = "serverConnections"; @@ -118,7 +104,7 @@ public void start(Buffer> buffer) { Thread.currentThread().interrupt(); throw new RuntimeException(ex); } - LOG.info("Started otel_trace_source on port " + oTelTraceSourceConfig.getPort() + "..."); + LOG.info("Started otel_trace_source on port {}...", oTelTraceSourceConfig.getPort()); } private SessionProtocol inferProtocolFromConfig() { @@ -148,50 +134,7 @@ private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { - ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics, oTelTraceSourceConfig.getRequestTimeoutInMillis()); - RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null - ? oTelTraceSourceConfig.getRetryInfo() - : DEFAULT_RETRY_INFO; - - // todo tlongo move creation of handler to ArmeriaHttpService - HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); - - configureAuthentication(serverBuilder); - - if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { - serverBuilder.annotatedService(httpService, httpExceptionHandler); - } else { - serverBuilder.annotatedService(httpService, DecodingService.newDecorator(), httpExceptionHandler); - } - } - - // todo tlongo move to http service -> Create additional layer. See GrpcService - private void configureAuthentication(ServerBuilder serverBuilder) { - if (oTelTraceSourceConfig.getAuthentication() == null || oTelTraceSourceConfig.getAuthentication().getPluginName().equals(UNAUTHENTICATED_PLUGIN_NAME)) { - LOG.warn("Creating otel_trace_source http service without authentication. This is not secure."); - LOG.warn("In order to set up Http Basic authentication for the otel-trace-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#authentication-configurations"); - } else { - ArmeriaHttpAuthenticationProvider authenticationProvider = createAuthenticationProvider(oTelTraceSourceConfig.getAuthentication()); - authenticationProvider.getAuthenticationDecorator().ifPresent(serverBuilder::decorator); - } - } - - // todo tlongo move to http service -> Create additional layer. See GrpcService - private ArmeriaHttpAuthenticationProvider createAuthenticationProvider(final PluginModel authenticationConfiguration) { - Map pluginSettings = authenticationConfiguration.getPluginSettings(); - - // controversial - // the world would be a nicer place, if mere configs were not be treated as plugins - // this method replaces the process of - // yaml -> pluginmodel -> pluginsettings -> configPojo -> pluginfactory -> provider - // with - // yaml -> configPojo -> provider (we could eliminate using Plugin* Classes all together by parsing the yaml section at startup, e.g. like retryInfo) - // pros: - // - we can easily reason about the origins of the provider - // - it becomes testable - // cons: - // - currently tied to one impl by using 'new'. - return new HttpBasicArmeriaHttpAuthenticationProvider(new HttpBasicAuthenticationConfig(pluginSettings.get("username").toString(), pluginSettings.get("password").toString())); + new HttpService(pluginMetrics, oTelTraceSourceConfig).create(serverBuilder, buffer); } private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index 14a3367d9d..742cd225bb 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -61,8 +61,8 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu } // todo tlongo healthcheck? - // todo tlongo authentication for http (Auth in Grpc Service is grpc specific) + // todo tlongo make path configurable @Post("/opentelemetry.proto.collector.trace.v1.TraceService/Export") @Consumes(value = "application/json") public ExportTraceServiceResponse exportTrace(ExportTraceServiceRequest request) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java new file mode 100644 index 0000000000..af5a777a0a --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java @@ -0,0 +1,81 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace.http; + +import static org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider.UNAUTHENTICATED_PLUGIN_NAME; + +import java.time.Duration; +import java.util.Map; + +import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; +import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; +import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; +import org.opensearch.dataprepper.plugins.source.oteltrace.RetryInfoConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.encoding.DecodingService; + +public class HttpService { + private static final Logger LOG = LoggerFactory.getLogger(HttpService.class); + // todo tlongo include in config + private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); + + private final PluginMetrics pluginMetrics; + private final OTelTraceSourceConfig oTelTraceSourceConfig; + + public HttpService(PluginMetrics pluginMetrics, OTelTraceSourceConfig oTelTraceSourceConfig) { + this.pluginMetrics = pluginMetrics; + this.oTelTraceSourceConfig = oTelTraceSourceConfig; + } + + public ArmeriaHttpService create(ServerBuilder serverBuilder, Buffer> buffer) { + RetryInfoConfig retryInfo = oTelTraceSourceConfig.getRetryInfo() != null + ? oTelTraceSourceConfig.getRetryInfo() + : DEFAULT_RETRY_INFO; + ArmeriaHttpService httpService = new ArmeriaHttpService(buffer, pluginMetrics, oTelTraceSourceConfig.getRequestTimeoutInMillis()); + HttpExceptionHandler httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, retryInfo.getMinDelay(), retryInfo.getMaxDelay()); + + configureAuthentication(serverBuilder); + + if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { + serverBuilder.annotatedService(httpService, httpExceptionHandler); + } else { + serverBuilder.annotatedService(httpService, DecodingService.newDecorator(), httpExceptionHandler); + } + + return httpService; + } + + private void configureAuthentication(ServerBuilder serverBuilder) { + if (oTelTraceSourceConfig.getAuthentication() == null || oTelTraceSourceConfig.getAuthentication().getPluginName().equals(UNAUTHENTICATED_PLUGIN_NAME)) { + LOG.warn("Creating otel_trace_source http service without authentication. This is not secure."); + LOG.warn("In order to set up Http Basic authentication for the otel-trace-source, go here: https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/otel-trace-source#authentication-configurations"); + } else { + ArmeriaHttpAuthenticationProvider authenticationProvider = createAuthenticationProvider(oTelTraceSourceConfig.getAuthentication()); + authenticationProvider.getAuthenticationDecorator().ifPresent(serverBuilder::decorator); + } + } + + private ArmeriaHttpAuthenticationProvider createAuthenticationProvider(final PluginModel authenticationConfiguration) { + Map pluginSettings = authenticationConfiguration.getPluginSettings(); + + // controversial + // the world would be a nicer place, if mere configs were not be treated as plugins + // this method replaces the process of + // yaml -> pluginmodel -> pluginsettings -> configPojo -> pluginfactory -> provider + // with + // yaml -> configPojo -> provider (we could eliminate using Plugin* Classes all together by parsing the yaml section at startup, e.g. like retryInfo) + // pros: + // - we can easily reason about the origins of the provider + // - it becomes testable + // cons: + // - currently tied to one impl by using 'new'. + return new HttpBasicArmeriaHttpAuthenticationProvider(new HttpBasicAuthenticationConfig(pluginSettings.get("username").toString(), pluginSettings.get("password").toString())); + } +} diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 23e6c39db6..0944d5f85d 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -344,31 +344,7 @@ void testGrpcRequestWithoutAuthentication_with_unsuccessful_response() throws Ex assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.UNAUTHENTICATED)); } - @Test - void testHttpWithoutSslFailsWhenSslIsEnabled() throws InvalidProtocolBufferException { - when(oTelTraceSourceConfig.isSsl()).thenReturn(true); - when(oTelTraceSourceConfig.getSslKeyCertChainFile()).thenReturn("data/certificate/test_cert.crt"); - when(oTelTraceSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); - configureObjectUnderTest(); - SOURCE.start(buffer); - - WebClient client = WebClient.builder("http://127.0.0.1:21890") - .build(); - - CompletionException exception = assertThrows(CompletionException.class, () -> client.execute(RequestHeaders.builder() - .scheme(SessionProtocol.HTTP) - .authority("127.0.0.1:21890") - .method(HttpMethod.POST) - .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") - .contentType(MediaType.JSON_UTF_8) - .build(), - HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) - .aggregate() - .join()); - - assertThat(exception.getCause(), instanceOf(ClosedSessionException.class)); - } - + @Test void testGrpcFailsIfSslIsEnabledAndNoTls() { when(oTelTraceSourceConfig.isSsl()).thenReturn(true); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 82d398df1a..7302661eb6 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -5,8 +5,11 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Named.named; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; @@ -23,10 +26,10 @@ import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; import java.nio.charset.StandardCharsets; -import java.time.Duration; import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Stream; @@ -63,6 +66,7 @@ import com.google.protobuf.util.JsonFormat; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; +import com.linecorp.armeria.common.ClosedSessionException; import com.linecorp.armeria.common.HttpData; import com.linecorp.armeria.common.HttpMethod; import com.linecorp.armeria.common.HttpStatus; @@ -85,7 +89,6 @@ @ExtendWith(MockitoExtension.class) class OTelTraceSource_HttpServiceTest { private static final String TEST_PIPELINE_NAME = "test_pipeline"; - private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); @Mock private ServerBuilder serverBuilder; @@ -147,7 +150,6 @@ void beforeEach() { when(oTelTraceSourceConfig.getMaxConnectionCount()).thenReturn(10); when(oTelTraceSourceConfig.getThreadCount()).thenReturn(5); when(oTelTraceSourceConfig.getCompression()).thenReturn(CompressionOption.NONE); - when(oTelTraceSourceConfig.getRetryInfo()).thenReturn(TEST_RETRY_INFO); // default: we don't want authentication when(oTelTraceSourceConfig.getAuthentication()).thenReturn(null); @@ -208,10 +210,35 @@ void request_that_causes_overflow_exception_should_not_be_written_to_buffer_and_ makeRequestAndAssertResponse("/opentelemetry.proto.collector.trace.v1.TraceService/Export", createExportTraceRequest(), (response, throwable) -> { assertThat(response.status(), is(HttpStatus.INSUFFICIENT_STORAGE)); - assertResponseBodyForRetryInformation(response); + assertResponseBodyForRetryInformation(response, "0.100s"); }); } + @Test + void request_over_http_with_ssl_enabled_fails() { + when(oTelTraceSourceConfig.isSsl()).thenReturn(true); + when(oTelTraceSourceConfig.getSslKeyCertChainFile()).thenReturn("data/certificate/test_cert.crt"); + when(oTelTraceSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); + configureObjectUnderTest(); + SOURCE.start(buffer); + + WebClient client = WebClient.builder("http://127.0.0.1:21890") + .build(); + + CompletionException exception = assertThrows(CompletionException.class, () -> client.execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), + HttpData.copyOf(JsonFormat.printer().print(createExportTraceRequest()).getBytes())) + .aggregate() + .join()); + + assertThat(exception.getCause(), instanceOf(ClosedSessionException.class)); + } + @ParameterizedTest @MethodSource("generateCredentials") void request_with_credentials_returns_expected_status_code(AuthTestDataHolder testData) throws InvalidProtocolBufferException { @@ -277,9 +304,6 @@ private void makeRequestAndAssertResponse(String path, ExportTraceServiceRequest .join(); } - - // todo tlongo https test - private ExportTraceServiceRequest createExportTraceRequest() { final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) @@ -296,10 +320,10 @@ private ExportTraceServiceRequest createExportTraceRequest() { .build(); } - private void assertResponseBodyForRetryInformation(final AggregatedHttpResponse response) { + private void assertResponseBodyForRetryInformation(final AggregatedHttpResponse response, String expectedDelay) { String body = response.content(StandardCharsets.UTF_8); - // todo tlongo assert delay value - assertThat(body, hasJsonPath("$.details[0].retryDelay")); + // todo tlongo map to numeric value when creating status in exception handler + assertThat(body, hasJsonPath("$.details[0].retryDelay", equalTo(expectedDelay))); } } From fa101abacc6b0c11314cfed43d02476ee2233ad5 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 20 Dec 2024 12:21:07 +0100 Subject: [PATCH 12/30] [WIP] Add pr description Signed-off-by: Tomas Longo --- pull_request_description.md | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 pull_request_description.md diff --git a/pull_request_description.md b/pull_request_description.md new file mode 100644 index 0000000000..77ba3fe41f --- /dev/null +++ b/pull_request_description.md @@ -0,0 +1,65 @@ +### Description + +Introduce http endpoint to support otl/http for the otel trace source. So far, this draft PR contains: + +- A single armeria server listening on a port +- A HTTP Service + - listening under `/opentelemetry.proto.collector.trace.v1.TraceService/Export` + - processing ExportTraceServiceRequest + - supporting basic auth/tls +- (not yet complete)Testing of the http service + +This draft PR serves as starting for further discussions: + +# Configuration of HTTP and gRPC Service + +Currently, the source config is responsible for setting up a working gRPC service. Now, an HTTP Service has to be configured as well. + +As far as this PR is concerned, we can assign the current config items into two groups: + +- Config items for gRPC specific features (e.g. `unframed_requests`, `proto_reflection_service`) +- Config items relevant for both services (e.g. `compression`, `authentication`) and should/could + +This leads to the questions how the structure of the config should look like in the future + +**Create distinct sections for every endpoint** + +```yaml +port: 123 +thread_count: 123 +... +http: + path: /path + compression: gzip + authentication: + http_basic: +grpc: + compression: gzip + proto_reflection_service: true + authentication: + http_basic: +``` + +**Let the endpoint share as much of the current config as possible** + +e.g. features like `compression`, `authentication` + + +# Do we want to keep unframed requests + +My understanding is that unframed requests enable clients to send plain http requests to the gRPC endpoint and would be rendered deprecated by this PR. However, there might be more to this feature which I'm currently unaware of. + + +### Issues Resolved +Resolves #4983 + +Related to #5259 + +### Check List +- [ ] New functionality includes testing. +- [ ] New functionality has a documentation issue. Please link to it in this PR. + - [ ] New functionality has javadoc added +- [ ] Commits are signed with a real name per the DCO + +By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. +For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md). From dec354e9b37b17a3e12c96a6c0071ad1a8537c79 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 10 Jan 2025 09:56:06 +0100 Subject: [PATCH 13/30] [WIP] Remove pr description Signed-off-by: Tomas Longo --- pull_request_description.md | 65 ------------------------------------- 1 file changed, 65 deletions(-) delete mode 100644 pull_request_description.md diff --git a/pull_request_description.md b/pull_request_description.md deleted file mode 100644 index 77ba3fe41f..0000000000 --- a/pull_request_description.md +++ /dev/null @@ -1,65 +0,0 @@ -### Description - -Introduce http endpoint to support otl/http for the otel trace source. So far, this draft PR contains: - -- A single armeria server listening on a port -- A HTTP Service - - listening under `/opentelemetry.proto.collector.trace.v1.TraceService/Export` - - processing ExportTraceServiceRequest - - supporting basic auth/tls -- (not yet complete)Testing of the http service - -This draft PR serves as starting for further discussions: - -# Configuration of HTTP and gRPC Service - -Currently, the source config is responsible for setting up a working gRPC service. Now, an HTTP Service has to be configured as well. - -As far as this PR is concerned, we can assign the current config items into two groups: - -- Config items for gRPC specific features (e.g. `unframed_requests`, `proto_reflection_service`) -- Config items relevant for both services (e.g. `compression`, `authentication`) and should/could - -This leads to the questions how the structure of the config should look like in the future - -**Create distinct sections for every endpoint** - -```yaml -port: 123 -thread_count: 123 -... -http: - path: /path - compression: gzip - authentication: - http_basic: -grpc: - compression: gzip - proto_reflection_service: true - authentication: - http_basic: -``` - -**Let the endpoint share as much of the current config as possible** - -e.g. features like `compression`, `authentication` - - -# Do we want to keep unframed requests - -My understanding is that unframed requests enable clients to send plain http requests to the gRPC endpoint and would be rendered deprecated by this PR. However, there might be more to this feature which I'm currently unaware of. - - -### Issues Resolved -Resolves #4983 - -Related to #5259 - -### Check List -- [ ] New functionality includes testing. -- [ ] New functionality has a documentation issue. Please link to it in this PR. - - [ ] New functionality has javadoc added -- [ ] Commits are signed with a real name per the DCO - -By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. -For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/data-prepper/blob/main/CONTRIBUTING.md). From 4f6312d7d433bc58424dd6979faa7f50b001c7d4 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 17 Jan 2025 12:44:55 +0100 Subject: [PATCH 14/30] [WIP] Fix checkstyle findings Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSourceTest.java | 3 -- .../OTelTraceSource_GrpcRequestTest.java | 31 ------------------- .../OTelTraceSource_HttpServiceTest.java | 2 +- 3 files changed, 1 insertion(+), 35 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 0944d5f85d..d319079ef0 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -13,7 +13,6 @@ import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; -import com.linecorp.armeria.common.ClosedSessionException; import com.linecorp.armeria.common.HttpData; import com.linecorp.armeria.common.HttpMethod; import com.linecorp.armeria.common.HttpStatus; @@ -85,12 +84,10 @@ import java.util.StringJoiner; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; -import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java index 64bf03e38e..fa2264ceae 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java @@ -13,53 +13,35 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; -import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.SSL; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; import java.util.Base64; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.StringJoiner; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.zip.GZIPOutputStream; -import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -72,13 +54,10 @@ import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.GrpcRequestExceptionHandler; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; -import org.opensearch.dataprepper.metrics.MetricNames; import org.opensearch.dataprepper.metrics.MetricsTestUtil; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; @@ -93,38 +72,28 @@ import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.health.HealthGrpcService; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.Clients; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpData; -import com.linecorp.armeria.common.HttpHeaderNames; import com.linecorp.armeria.common.HttpMethod; import com.linecorp.armeria.common.HttpStatus; import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; import com.linecorp.armeria.server.grpc.GrpcServiceBuilder; -import com.linecorp.armeria.server.healthcheck.HealthCheckService; import io.grpc.BindableService; -import io.grpc.ServerServiceDefinition; import io.grpc.Status; import io.grpc.StatusRuntimeException; -import io.micrometer.core.instrument.Measurement; -import io.micrometer.core.instrument.Statistic; import io.netty.util.AsciiString; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceResponse; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 7302661eb6..66ee1f9025 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.source.oteltrace; +import static com.jayway.jsonpath.matchers.JsonPathMatchers.hasJsonPath; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.MatcherAssert.assertThat; @@ -84,7 +85,6 @@ import io.opentelemetry.proto.trace.v1.ResourceSpans; import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; -import static com.jayway.jsonpath.matchers.JsonPathMatchers.*; @ExtendWith(MockitoExtension.class) class OTelTraceSource_HttpServiceTest { From 6c6fbff89b2338d741c328a5a1d66c59c19b8293 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 24 Jan 2025 15:09:56 +0100 Subject: [PATCH 15/30] [WIP] Fix issue with http service being enabled, while grpc service accepts unframed requests Signed-off-by: Tomas Longo --- .../otelmetrics/OTelMetricsSourceTest.java | 31 +++++++++---------- .../source/oteltrace/OTelTraceSource.java | 5 ++- .../source/oteltrace/http/HttpService.java | 1 + .../source/oteltrace/OTelTraceSourceTest.java | 9 +++++- .../OTelTraceSource_GrpcRequestTest.java | 6 ++-- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java index 81214e3c10..8e41255b0e 100644 --- a/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java +++ b/data-prepper-plugins/otel-metrics-source/src/test/java/org/opensearch/dataprepper/plugins/source/otelmetrics/OTelMetricsSourceTest.java @@ -350,7 +350,6 @@ void testHttpFullJsonWithCustomPathAndUnframedRequests() throws InvalidProtocolB .join(); } - @Test void testHttpFullJsonWithCustomPathAndAuthHeader_with_successful_response() throws InvalidProtocolBufferException { when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); @@ -420,7 +419,7 @@ void testHttpRequestWithInvalidCredentials_with_unsuccessful_response() throws I when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); when(httpBasicAuthenticationConfig.getPassword()).thenReturn(PASSWORD); final GrpcAuthenticationProvider grpcAuthenticationProvider = new GrpcBasicAuthenticationProvider(httpBasicAuthenticationConfig); - + when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) .thenReturn(grpcAuthenticationProvider); when(oTelMetricsSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", @@ -430,17 +429,17 @@ void testHttpRequestWithInvalidCredentials_with_unsuccessful_response() throws I ))); when(oTelMetricsSourceConfig.enableUnframedRequests()).thenReturn(true); when(oTelMetricsSourceConfig.getPath()).thenReturn(TEST_PATH); - + configureObjectUnderTest(); SOURCE.start(buffer); - + final String invalidUsername = "wrong_user"; final String invalidPassword = "wrong_password"; final String invalidCredentials = Base64.getEncoder() .encodeToString(String.format("%s:%s", invalidUsername, invalidPassword).getBytes(StandardCharsets.UTF_8)); - + final String transformedPath = "/" + TEST_PIPELINE_NAME + "/v1/metrics"; - + WebClient.of().prepare() .post("http://127.0.0.1:21891" + transformedPath) .content(MediaType.JSON_UTF_8, JsonFormat.printer().print(createExportMetricsRequest()).getBytes()) @@ -450,7 +449,7 @@ void testHttpRequestWithInvalidCredentials_with_unsuccessful_response() throws I .whenComplete((response, throwable) -> assertSecureResponseWithStatusCode(response, HttpStatus.UNAUTHORIZED, throwable)) .join(); } - + @Test void testGrpcRequestWithInvalidCredentials_with_unsuccessful_response() throws Exception { when(httpBasicAuthenticationConfig.getUsername()).thenReturn(USERNAME); @@ -489,10 +488,10 @@ void testHttpWithoutSslFailsWhenSslIsEnabled() throws InvalidProtocolBufferExcep when(oTelMetricsSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); configureObjectUnderTest(); SOURCE.start(buffer); - + WebClient client = WebClient.builder("http://127.0.0.1:21891") .build(); - + CompletionException exception = assertThrows(CompletionException.class, () -> client.execute(RequestHeaders.builder() .scheme(SessionProtocol.HTTP) .authority("127.0.0.1:21891") @@ -503,10 +502,10 @@ void testHttpWithoutSslFailsWhenSslIsEnabled() throws InvalidProtocolBufferExcep HttpData.copyOf(JsonFormat.printer().print(createExportMetricsRequest()).getBytes())) .aggregate() .join()); - + assertThat(exception.getCause(), instanceOf(ClosedSessionException.class)); } - + @Test void testGrpcFailsIfSslIsEnabledAndNoTls() { when(oTelMetricsSourceConfig.isSsl()).thenReturn(true); @@ -514,17 +513,17 @@ void testGrpcFailsIfSslIsEnabledAndNoTls() { when(oTelMetricsSourceConfig.getSslKeyFile()).thenReturn("data/certificate/test_decrypted_key.key"); configureObjectUnderTest(); SOURCE.start(buffer); - + MetricsServiceGrpc.MetricsServiceBlockingStub client = Clients.builder(GRPC_ENDPOINT) .build(MetricsServiceGrpc.MetricsServiceBlockingStub.class); - + StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportMetricsRequest())); - + assertThat(actualException.getStatus(), notNullValue()); assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.UNKNOWN)); } - - + + @Test void testServerStartCertFileSuccess() throws IOException { try (MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 26647b30bd..852e93daae 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -90,7 +90,10 @@ public void start(Buffer> buffer) { configureTaskExecutor(serverBuilder); configureGrpcService(serverBuilder, buffer); - configureHttpService(serverBuilder, buffer); + // todo tlongo needed until clarified if unframedRequests should survive + if (!oTelTraceSourceConfig.enableUnframedRequests()) { + configureHttpService(serverBuilder, buffer); + } server = serverBuilder.build(); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java index af5a777a0a..40578bdaa5 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java @@ -76,6 +76,7 @@ private ArmeriaHttpAuthenticationProvider createAuthenticationProvider(final Plu // - it becomes testable // cons: // - currently tied to one impl by using 'new'. + // todo tlongo clarify if simplifying config handling is something worth considering in this PR return new HttpBasicArmeriaHttpAuthenticationProvider(new HttpBasicAuthenticationConfig(pluginSettings.get("username").toString(), pluginSettings.get("password").toString())); } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index d319079ef0..3cea68d17b 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -40,6 +40,7 @@ import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -601,10 +602,15 @@ void testOptionalHttpAuthServiceNotInPlace() { } @Test - void testOptionalHttpAuthServiceInPlace() { + @Disabled + void testOptionalHttpAuthServiOceInPlace() { final Optional> function = Optional.of(httpService -> httpService); final Map settingsMap = new HashMap<>(); + // todo tlongo: Providing this pipeline config to data prepper let's it refuse to boot right away. Why is this test green, then? + // Because the it's the plugin factory that stumbles over the missing plugin settings. But, the plugin factory is + // mocked in this whole test class, always returning a GrpcBasicAuthenticationProvider. IMO this test does not + // service its purpose settingsMap.put("authentication", new PluginModel("test", null)); settingsMap.put("unauthenticated_health_check", true); @@ -628,6 +634,7 @@ void testOptionalHttpAuthServiceInPlace() { } @Test + @Disabled // todo tlongo see testOptionalHttpAuthServiOceInPlace on why I disabled this test void testOptionalHttpAuthServiceInPlaceWithUnauthenticatedDisabled() { final Optional> function = Optional.of(httpService -> httpService); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java index fa2264ceae..9393340088 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java @@ -270,7 +270,7 @@ void gRPC_request_with_custom_path_throws_when_written_to_default_path() { final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(createExportTraceRequest())); assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); + assertThat(actualException.getMessage(), actualException.getStatus().getCode(), equalTo(Status.UNIMPLEMENTED.getCode())); } @ParameterizedTest @@ -291,7 +291,7 @@ void gRPC_request_returns_expected_status_for_exceptions_from_buffer( final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportTraceRequest)); assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(expectedStatusCode)); + assertThat(actualException.getMessage(), actualException.getStatus().getCode(), equalTo(expectedStatusCode)); } @Test @@ -306,7 +306,7 @@ void gRPC_request_throws_InvalidArgument_for_malformed_trace_data() { final StatusRuntimeException actualException = assertThrows(StatusRuntimeException.class, () -> client.export(exportTraceRequest)); assertThat(actualException.getStatus(), notNullValue()); - assertThat(actualException.getStatus().getCode(), equalTo(Status.Code.INVALID_ARGUMENT)); + assertThat(actualException.getMessage(), actualException.getStatus().getCode(), equalTo(Status.Code.INVALID_ARGUMENT)); verifyNoInteractions(buffer); } From 59003ab609823e7d424ba5f8801a3d220125ff46 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 10:21:07 +0100 Subject: [PATCH 16/30] Refactor EndToEndRawSpanTest Signed-off-by: Tomas Longo --- e2e-test/trace/build.gradle | 1 + .../trace/EndToEndRawSpanTest.java | 94 +++++++++++-------- 2 files changed, 56 insertions(+), 39 deletions(-) diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index b46882be93..30c243ff05 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -207,6 +207,7 @@ dependencies { integrationTestImplementation project(':data-prepper-plugins:aws-plugin-api') integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-processor') integrationTestImplementation testLibs.awaitility + integrationTestImplementation "org.assertj:assertj-core:3.25.3" integrationTestImplementation "io.opentelemetry.proto:opentelemetry-proto:${targetOpenTelemetryVersion}" integrationTestImplementation libs.protobuf.util integrationTestImplementation libs.armeria.core diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index 829b8e8e6b..157de7920f 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -42,12 +42,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; public class EndToEndRawSpanTest { private static final int DATA_PREPPER_PORT_1 = 21890; @@ -87,7 +87,58 @@ public class EndToEndRawSpanTest { @Test public void testPipelineEndToEnd() { - //Send data to otel trace source + final List> expectedDocuments = sendTracesToOpenSearchAndReturnExpectedDocuments(); + final RestHighLevelClient restHighLevelClient = createRestClientForSearch(); + final SearchSourceBuilder sourceBuilder = createSearchSourceBuilder(); + final SearchRequest searchRequest = new SearchRequest(INDEX_NAME).source(sourceBuilder); + + // Wait for data to flow through pipeline and be indexed by ES + await().atLeast(3, TimeUnit.SECONDS).atMost(30, TimeUnit.SECONDS).untilAsserted( + () -> { + refreshIndices(restHighLevelClient); + final SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + final List> foundSources = getSourcesFromSearchHits(searchResponse.getHits()); + + assertThat(expectedDocuments).hasSize(foundSources.size()); + assertThatFoundDocumentsContainAllFieldsFromExpectedDocuments(expectedDocuments, foundSources); + } + ); + } + + private void assertThatFoundDocumentsContainAllFieldsFromExpectedDocuments(List> expectedDocuments, List> foundDocuments) { + /** + * Our raw trace prepper add more fields than the actual sent object. These are defaults from the proto. + * So assertion is done if all the expected fields exists. + * + * TODO: Can we do better? + */ + expectedDocuments.forEach(expectedDoc -> { + Set> foundEntrySet = foundDocuments.stream() + .filter(i -> i.get("spanId").equals(expectedDoc.get("spanId"))) + .findFirst().get() + .entrySet(); + + assertThat(foundEntrySet).containsAll(expectedDoc.entrySet()); + }); + + } + + private SearchSourceBuilder createSearchSourceBuilder() { + return SearchSourceBuilder.searchSource() + .size(100) + .fetchField(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD) + .fetchField(TraceGroup.TRACE_GROUP_END_TIME_FIELD, "strict_date_time") + .fetchField(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD); + } + + private RestHighLevelClient createRestClientForSearch() { + return new ConnectionConfiguration.Builder(Collections.singletonList("https://127.0.0.1:9200")) + .withUsername("admin") + .withPassword("admin") + .build().createClient(null); + } + + private List> sendTracesToOpenSearchAndReturnExpectedDocuments() { final ExportTraceServiceRequest exportTraceServiceRequestTrace1BatchWithRoot = getExportTraceServiceRequest( getResourceSpansBatch(TEST_SPAN_SET_1_WITH_ROOT_SPAN) ); @@ -107,44 +158,9 @@ public void testPipelineEndToEnd() { sendExportTraceServiceRequestToSource(DATA_PREPPER_PORT_2, exportTraceServiceRequestTrace1BatchNoRoot); //Verify data in OpenSearch backend - final List> expectedDocuments = getExpectedDocuments( + return getExpectedDocuments( exportTraceServiceRequestTrace1BatchWithRoot, exportTraceServiceRequestTrace1BatchNoRoot, exportTraceServiceRequestTrace2BatchWithRoot, exportTraceServiceRequestTrace2BatchNoRoot); - final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( - Collections.singletonList("https://127.0.0.1:9200")); - builder.withUsername("admin"); - builder.withPassword("admin"); - final RestHighLevelClient restHighLevelClient = builder.build().createClient(null); - // Wait for data to flow through pipeline and be indexed by ES - await().atLeast(3, TimeUnit.SECONDS).atMost(20, TimeUnit.SECONDS).untilAsserted( - () -> { - refreshIndices(restHighLevelClient); - final SearchRequest searchRequest = new SearchRequest(INDEX_NAME); - searchRequest.source( - SearchSourceBuilder.searchSource() - .size(100) - .fetchField(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD) - .fetchField(TraceGroup.TRACE_GROUP_END_TIME_FIELD, "strict_date_time") - .fetchField(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD) - ); - final SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - final List> foundSources = getSourcesFromSearchHits(searchResponse.getHits()); - assertEquals(expectedDocuments.size(), foundSources.size()); - /** - * Our raw trace prepper add more fields than the actual sent object. These are defaults from the proto. - * So assertion is done if all the expected fields exists. - * - * TODO: Can we do better? - * - */ - expectedDocuments.forEach(expectedDoc -> { - assertTrue(foundSources.stream() - .filter(i -> i.get("spanId").equals(expectedDoc.get("spanId"))) - .findFirst().get() - .entrySet().containsAll(expectedDoc.entrySet())); - }); - } - ); } private void refreshIndices(final RestHighLevelClient restHighLevelClient) throws IOException { From f4c6fcf6b19b8448151bdce5e087c2cad5d43cbb Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 11:27:51 +0100 Subject: [PATCH 17/30] Create ArmeriaAuthenticationProvider via PluginFactory Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 2 +- .../source/oteltrace/http/HttpService.java | 23 +++++-------------- .../source/oteltrace/OTelTraceSourceTest.java | 8 ++++++- .../OTelTraceSource_HttpServiceTest.java | 3 +++ 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 852e93daae..95bafcce01 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -137,7 +137,7 @@ private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { - new HttpService(pluginMetrics, oTelTraceSourceConfig).create(serverBuilder, buffer); + new HttpService(pluginMetrics, oTelTraceSourceConfig, pluginFactory).create(serverBuilder, buffer); } private void configureHeadersAndHealthCheck(ServerBuilder serverBuilder) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java index 40578bdaa5..8ecc43d7cb 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java @@ -6,12 +6,12 @@ import java.util.Map; import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; -import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.model.configuration.PluginSetting; +import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.RetryInfoConfig; @@ -28,10 +28,12 @@ public class HttpService { private final PluginMetrics pluginMetrics; private final OTelTraceSourceConfig oTelTraceSourceConfig; + private final PluginFactory pluginFactory; - public HttpService(PluginMetrics pluginMetrics, OTelTraceSourceConfig oTelTraceSourceConfig) { + public HttpService(PluginMetrics pluginMetrics, OTelTraceSourceConfig oTelTraceSourceConfig, PluginFactory pluginFactory) { this.pluginMetrics = pluginMetrics; this.oTelTraceSourceConfig = oTelTraceSourceConfig; + this.pluginFactory = pluginFactory; } public ArmeriaHttpService create(ServerBuilder serverBuilder, Buffer> buffer) { @@ -64,19 +66,6 @@ private void configureAuthentication(ServerBuilder serverBuilder) { private ArmeriaHttpAuthenticationProvider createAuthenticationProvider(final PluginModel authenticationConfiguration) { Map pluginSettings = authenticationConfiguration.getPluginSettings(); - - // controversial - // the world would be a nicer place, if mere configs were not be treated as plugins - // this method replaces the process of - // yaml -> pluginmodel -> pluginsettings -> configPojo -> pluginfactory -> provider - // with - // yaml -> configPojo -> provider (we could eliminate using Plugin* Classes all together by parsing the yaml section at startup, e.g. like retryInfo) - // pros: - // - we can easily reason about the origins of the provider - // - it becomes testable - // cons: - // - currently tied to one impl by using 'new'. - // todo tlongo clarify if simplifying config handling is something worth considering in this PR - return new HttpBasicArmeriaHttpAuthenticationProvider(new HttpBasicAuthenticationConfig(pluginSettings.get("username").toString(), pluginSettings.get("password").toString())); + return pluginFactory.loadPlugin(ArmeriaHttpAuthenticationProvider.class, new PluginSetting(authenticationConfiguration.getPluginName(), pluginSettings)); } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 3cea68d17b..b941d38e55 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -50,6 +50,7 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.GrpcRequestExceptionHandler; +import org.opensearch.dataprepper.armeria.authentication.ArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.GrpcAuthenticationProvider; import org.opensearch.dataprepper.armeria.authentication.HttpBasicAuthenticationConfig; import org.opensearch.dataprepper.metrics.MetricNames; @@ -63,6 +64,7 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.types.ByteCount; import org.opensearch.dataprepper.plugins.GrpcBasicAuthenticationProvider; +import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; @@ -154,6 +156,9 @@ class OTelTraceSourceTest { @Mock private GrpcBasicAuthenticationProvider authenticationProvider; + @Mock + private HttpBasicArmeriaHttpAuthenticationProvider armeriaHttpAuthenticationProvider; + @Mock(lenient = true) private OTelTraceSourceConfig oTelTraceSourceConfig; @@ -163,7 +168,6 @@ class OTelTraceSourceTest { @Mock private HttpBasicAuthenticationConfig httpBasicAuthenticationConfig; - private PluginSetting pluginSetting; private PluginSetting testPluginSetting; private PluginMetrics pluginMetrics; private PipelineDescription pipelineDescription; @@ -199,6 +203,8 @@ void beforeEach() { lenient().when(pluginFactory.loadPlugin(eq(GrpcAuthenticationProvider.class), any(PluginSetting.class))) .thenReturn(authenticationProvider); + lenient().when(pluginFactory.loadPlugin(eq(ArmeriaHttpAuthenticationProvider.class), any(PluginSetting.class))) + .thenReturn(armeriaHttpAuthenticationProvider); configureObjectUnderTest(); pipelineDescription = mock(PipelineDescription.class); lenient().when(pipelineDescription.getPipelineName()).thenReturn(TEST_PIPELINE_NAME); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 66ee1f9025..5f030b50f5 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -60,6 +60,7 @@ import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.HttpBasicArmeriaHttpAuthenticationProvider; import org.opensearch.dataprepper.plugins.codec.CompressionOption; import com.google.protobuf.ByteString; @@ -243,6 +244,8 @@ void request_over_http_with_ssl_enabled_fails() { @MethodSource("generateCredentials") void request_with_credentials_returns_expected_status_code(AuthTestDataHolder testData) throws InvalidProtocolBufferException { when(oTelTraceSourceConfig.getAuthentication()).thenReturn(new PluginModel("http_basic", Map.of("username", PROVIDED_CONFIG.getUsername(), "password",PROVIDED_CONFIG.getPassword()))); + HttpBasicArmeriaHttpAuthenticationProvider authProvider = new HttpBasicArmeriaHttpAuthenticationProvider(new HttpBasicAuthenticationConfig(PROVIDED_CONFIG.getUsername(), PROVIDED_CONFIG.getPassword())); + lenient().when(pluginFactory.loadPlugin(eq(ArmeriaHttpAuthenticationProvider.class), any(PluginSetting.class))).thenReturn(authProvider); SOURCE.start(buffer); makeRequestWithCredentialsAndAssertResponse("/opentelemetry.proto.collector.trace.v1.TraceService/Export", From cf3b2e7066ee9db8667fa42e9d413d39e522a3b0 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 11:35:01 +0100 Subject: [PATCH 18/30] Rename GrpcRetryInfoCalculator Signed-off-by: Tomas Longo --- .../GrpcRequestExceptionHandler.java | 4 ++-- ...alculator.java => RetryInfoCalculator.java} | 5 ++--- .../GrpcRetryInfoCalculatorTest.java | 18 +++++++++--------- .../oteltrace/http/HttpExceptionHandler.java | 6 +++--- 4 files changed, 16 insertions(+), 17 deletions(-) rename data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/{GrpcRetryInfoCalculator.java => RetryInfoCalculator.java} (92%) diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java index 1b7f591f24..964bb36373 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRequestExceptionHandler.java @@ -39,14 +39,14 @@ public class GrpcRequestExceptionHandler implements GoogleGrpcExceptionHandlerFu private final Counter badRequestsCounter; private final Counter requestsTooLargeCounter; private final Counter internalServerErrorCounter; - private final GrpcRetryInfoCalculator retryInfoCalculator; + private final RetryInfoCalculator retryInfoCalculator; public GrpcRequestExceptionHandler(final PluginMetrics pluginMetrics, Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { requestTimeoutsCounter = pluginMetrics.counter(REQUEST_TIMEOUTS); badRequestsCounter = pluginMetrics.counter(BAD_REQUESTS); requestsTooLargeCounter = pluginMetrics.counter(REQUESTS_TOO_LARGE); internalServerErrorCounter = pluginMetrics.counter(INTERNAL_SERVER_ERROR); - retryInfoCalculator = new GrpcRetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); + retryInfoCalculator = new RetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); } @Override diff --git a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/RetryInfoCalculator.java similarity index 92% rename from data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java rename to data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/RetryInfoCalculator.java index 6c7e3d8bf0..33c4af3f07 100644 --- a/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/GrpcRetryInfoCalculator.java +++ b/data-prepper-plugins/armeria-common/src/main/java/org/opensearch/dataprepper/RetryInfoCalculator.java @@ -6,8 +6,7 @@ import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; -// todo tlongo rename -public class GrpcRetryInfoCalculator { +public class RetryInfoCalculator { private final Duration minimumDelay; private final Duration maximumDelay; @@ -15,7 +14,7 @@ public class GrpcRetryInfoCalculator { private final AtomicReference lastTimeCalled; private final AtomicReference nextDelay; - public GrpcRetryInfoCalculator(Duration minimumDelay, Duration maximumDelay) { + public RetryInfoCalculator(Duration minimumDelay, Duration maximumDelay) { this.minimumDelay = minimumDelay; this.maximumDelay = maximumDelay; // Create a cushion so that the calculator treats a first quick exception (after prepper startup) as normal request (e.g. does not calculate a backoff) diff --git a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java index 5611826ef7..5cd79a3c1b 100644 --- a/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java +++ b/data-prepper-plugins/armeria-common/src/test/java/org/opensearch/dataprepper/GrpcRetryInfoCalculatorTest.java @@ -12,7 +12,7 @@ public class GrpcRetryInfoCalculatorTest { @Test public void testMinimumDelayOnFirstCall() { - RetryInfo retryInfo = new GrpcRetryInfoCalculator(Duration.ofMillis(100), Duration.ofSeconds(1)).createRetryInfo(); + RetryInfo retryInfo = new RetryInfoCalculator(Duration.ofMillis(100), Duration.ofSeconds(1)).createRetryInfo(); assertThat(retryInfo.getRetryDelay().getNanos(), equalTo(100_000_000)); assertThat(retryInfo.getRetryDelay().getSeconds(), equalTo(0L)); @@ -20,8 +20,8 @@ public void testMinimumDelayOnFirstCall() { @Test public void testExponentialBackoff() { - GrpcRetryInfoCalculator calculator = - new GrpcRetryInfoCalculator(Duration.ofSeconds(1), Duration.ofSeconds(10)); + RetryInfoCalculator calculator = + new RetryInfoCalculator(Duration.ofSeconds(1), Duration.ofSeconds(10)); RetryInfo retryInfo1 = calculator.createRetryInfo(); RetryInfo retryInfo2 = calculator.createRetryInfo(); RetryInfo retryInfo3 = calculator.createRetryInfo(); @@ -35,8 +35,8 @@ public void testExponentialBackoff() { @Test public void testUsesMaximumAsLongestDelay() { - GrpcRetryInfoCalculator calculator = - new GrpcRetryInfoCalculator(Duration.ofSeconds(1), Duration.ofSeconds(2)); + RetryInfoCalculator calculator = + new RetryInfoCalculator(Duration.ofSeconds(1), Duration.ofSeconds(2)); RetryInfo retryInfo1 = calculator.createRetryInfo(); RetryInfo retryInfo2 = calculator.createRetryInfo(); RetryInfo retryInfo3 = calculator.createRetryInfo(); @@ -49,8 +49,8 @@ public void testUsesMaximumAsLongestDelay() { @Test public void testResetAfterDelayWearsOff() throws InterruptedException { int minDelayNanos = 1_000_000; - GrpcRetryInfoCalculator calculator = - new GrpcRetryInfoCalculator(Duration.ofNanos(minDelayNanos), Duration.ofSeconds(1)); + RetryInfoCalculator calculator = + new RetryInfoCalculator(Duration.ofNanos(minDelayNanos), Duration.ofSeconds(1)); RetryInfo retryInfo1 = calculator.createRetryInfo(); RetryInfo retryInfo2 = calculator.createRetryInfo(); @@ -66,8 +66,8 @@ public void testResetAfterDelayWearsOff() throws InterruptedException { @Test public void testQuickFirstExceptionDoesNotTriggerBackoffCalculationEvenWithLongMinDelay() throws InterruptedException { - GrpcRetryInfoCalculator calculator = - new GrpcRetryInfoCalculator(Duration.ofSeconds(10), Duration.ofSeconds(20)); + RetryInfoCalculator calculator = + new RetryInfoCalculator(Duration.ofSeconds(10), Duration.ofSeconds(20)); RetryInfo retryInfo1 = calculator.createRetryInfo(); RetryInfo retryInfo2 = calculator.createRetryInfo(); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java index bdd69608bd..d0468a3079 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java @@ -4,7 +4,7 @@ import java.time.Duration; import java.util.concurrent.TimeoutException; -import org.opensearch.dataprepper.GrpcRetryInfoCalculator; +import org.opensearch.dataprepper.RetryInfoCalculator; import org.opensearch.dataprepper.exceptions.BadRequestException; import org.opensearch.dataprepper.exceptions.BufferWriteException; import org.opensearch.dataprepper.exceptions.RequestCancelledException; @@ -43,14 +43,14 @@ public class HttpExceptionHandler implements ExceptionHandlerFunction { private final Counter badRequestsCounter; private final Counter requestsTooLargeCounter; private final Counter internalServerErrorCounter; - private final GrpcRetryInfoCalculator retryInfoCalculator; + private final RetryInfoCalculator retryInfoCalculator; public HttpExceptionHandler(final PluginMetrics pluginMetrics, Duration retryInfoMinDelay, Duration retryInfoMaxDelay) { requestTimeoutsCounter = pluginMetrics.counter(REQUEST_TIMEOUTS); badRequestsCounter = pluginMetrics.counter(BAD_REQUESTS); requestsTooLargeCounter = pluginMetrics.counter(REQUESTS_TOO_LARGE); internalServerErrorCounter = pluginMetrics.counter(INTERNAL_SERVER_ERROR); - this.retryInfoCalculator = new GrpcRetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); + this.retryInfoCalculator = new RetryInfoCalculator(retryInfoMinDelay, retryInfoMaxDelay); } @Override From c35d9105a433bfb1b7bedcf8331b8a2388a86bf9 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 11:45:28 +0100 Subject: [PATCH 19/30] Create test for invalid payload Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSource.java | 2 +- .../source/oteltrace/grpc/GrpcService.java | 6 +--- .../OTelTraceSource_GrpcRequestTest.java | 3 +- .../OTelTraceSource_HttpServiceTest.java | 36 ++++++++++++++++++- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 95bafcce01..654d26416c 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -127,7 +127,7 @@ private void handleExecutionException(ExecutionException ex) { } private void configureGrpcService(ServerBuilder serverBuilder, Buffer> buffer) { - com.linecorp.armeria.server.grpc.GrpcService grpcService = new GrpcService(pluginFactory, oTelTraceSourceConfig, pluginMetrics, pipelineName, certificateProviderFactory).create(buffer, serverBuilder); + com.linecorp.armeria.server.grpc.GrpcService grpcService = new GrpcService(pluginFactory, oTelTraceSourceConfig, pluginMetrics, pipelineName).create(buffer, serverBuilder); if (CompressionOption.NONE.equals(oTelTraceSourceConfig.getCompression())) { serverBuilder.service(grpcService); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index 1eff3287b2..a5ae191a5a 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -43,22 +43,18 @@ public class GrpcService { private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); private static final String PIPELINE_NAME_PLACEHOLDER = "${pipelineName}"; - private static final String HTTP_HEALTH_CHECK_PATH = "/health"; public static final String REGEX_HEALTH = "regex:^/(?!health$).*$"; private final OTelTraceSourceConfig oTelTraceSourceConfig; private final GrpcAuthenticationProvider authenticationProvider; private final PluginMetrics pluginMetrics; private final String pipelineName; - // todo tlongo check why unused - private final CertificateProviderFactory certificateProviderFactory; - public GrpcService(PluginFactory pluginFactory, OTelTraceSourceConfig oTelTraceSourceConfig, PluginMetrics pluginMetrics, String pipelineName, CertificateProviderFactory certificateProviderFactory) { + public GrpcService(PluginFactory pluginFactory, OTelTraceSourceConfig oTelTraceSourceConfig, PluginMetrics pluginMetrics, String pipelineName) { this.oTelTraceSourceConfig = oTelTraceSourceConfig; this.pluginMetrics = pluginMetrics; this.pipelineName = pipelineName; this.authenticationProvider = createAuthenticationProvider(pluginFactory, oTelTraceSourceConfig); - this.certificateProviderFactory = certificateProviderFactory; } public com.linecorp.armeria.server.grpc.GrpcService create(Buffer> buffer, ServerBuilder serverBuilder) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java index 9393340088..d4ab75ce8c 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java @@ -169,8 +169,7 @@ void beforeEach() { lenient().when(grpcServiceBuilder.addService(any(BindableService.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useClientTimeoutHeader(anyBoolean())).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.useBlockingTaskExecutor(anyBoolean())).thenReturn(grpcServiceBuilder); - lenient().when(grpcServiceBuilder.exceptionHandler(any( - GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); + lenient().when(grpcServiceBuilder.exceptionHandler(any(GrpcRequestExceptionHandler.class))).thenReturn(grpcServiceBuilder); lenient().when(grpcServiceBuilder.build()).thenReturn(grpcService); lenient().when(authenticationProvider.getHttpAuthenticationService()).thenCallRealMethod(); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 5f030b50f5..50d6685a8a 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_PORT; import static org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig.DEFAULT_REQUEST_TIMEOUT_MS; @@ -174,7 +175,24 @@ private void configureObjectUnderTest() { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } - // todo tlongo add test for invalid payload + @Test + void request_fails_because_of_invalid_payload() throws Exception { + ExportTraceServiceRequest request = createInvalidExportTraceRequest(); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.POST) + .path("/opentelemetry.proto.collector.trace.v1.TraceService/Export") + .contentType(MediaType.JSON_UTF_8) + .build(), HttpData.copyOf(JsonFormat.printer().print(request).getBytes())) + .aggregate() + .whenComplete((response, throwable) -> assertThat(response.status(), is(HttpStatus.BAD_REQUEST))) + .join(); + + verifyNoInteractions(buffer); + } @Test void request_that_is_successful() throws Exception { @@ -307,6 +325,22 @@ private void makeRequestAndAssertResponse(String path, ExportTraceServiceRequest .join(); } + private ExportTraceServiceRequest createInvalidExportTraceRequest() { + final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() +// .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) +// .setSpanId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) + .setName(UUID.randomUUID().toString()) + .setKind(Span.SpanKind.SPAN_KIND_SERVER) + .setStartTimeUnixNano(100) + .setEndTimeUnixNano(101) + .setTraceState("SUCCESS").build(); + + return ExportTraceServiceRequest.newBuilder() + .addResourceSpans(ResourceSpans.newBuilder() + .addScopeSpans(ScopeSpans.newBuilder().addSpans(testSpan)).build()) + .build(); + } + private ExportTraceServiceRequest createExportTraceRequest() { final io.opentelemetry.proto.trace.v1.Span testSpan = Span.newBuilder() .setTraceId(ByteString.copyFromUtf8(UUID.randomUUID().toString())) From 19d2e9999b91d72c8c98b3ce3da9847e40460cf4 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 12:01:13 +0100 Subject: [PATCH 20/30] Add test for healthcheck Signed-off-by: Tomas Longo --- .../plugins/source/oteltrace/http/ArmeriaHttpService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index 742cd225bb..42f20644a6 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -60,8 +60,6 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu requestProcessDuration = pluginMetrics.timer(REQUEST_PROCESS_DURATION); } - // todo tlongo healthcheck? - // todo tlongo make path configurable @Post("/opentelemetry.proto.collector.trace.v1.TraceService/Export") @Consumes(value = "application/json") @@ -74,7 +72,6 @@ public ExportTraceServiceResponse exportTrace(ExportTraceServiceRequest request) return ExportTraceServiceResponse.newBuilder().build(); } - // todo tlongo exract in order to be used by http and grpc? private void processRequest(final ExportTraceServiceRequest request) { final Collection spans; From f56079f48d5fa74ade8c274eabe8a4ae8b321cff Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 13:03:00 +0100 Subject: [PATCH 21/30] Add test for http exception handler Signed-off-by: Tomas Longo --- .../source/oteltrace/grpc/GrpcService.java | 2 - .../oteltrace/http/ArmeriaHttpService.java | 4 - .../oteltrace/http/HttpExceptionHandler.java | 2 - .../source/oteltrace/http/HttpService.java | 1 - .../OTelTraceSource_HttpServiceTest.java | 21 +++- .../http/HttpExceptionHandlerTest.java | 97 +++++++++++++++++++ 6 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandlerTest.java diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index a5ae191a5a..5dc9723236 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -19,7 +19,6 @@ import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.RetryInfoConfig; -import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,7 +83,6 @@ public com.linecorp.armeria.server.grpc.GrpcService create(Buffer grpcServiceBuilder.addService(ServerInterceptors.intercept(oTelTraceGrpcService, serverInterceptors)); } - // todo tlongo extract into separate grpc config. Can't we have only one healthcheck for the whole server? We are already configuring one OtelTraceSource if (oTelTraceSourceConfig.hasHealthCheck()) { LOG.info("Health check is enabled"); grpcServiceBuilder.addService(new HealthGrpcService()); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index 42f20644a6..a16a301c60 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -108,9 +108,5 @@ private void processRequest(final ExportTraceServiceRequest request) { } successRequestsCounter.increment(); - - // todo tlongo what is the responseObserver used for? -// responseObserver.onNext(ExportTraceServiceResponse.newBuilder().build()); -// responseObserver.onCompleted(); } } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java index d0468a3079..c06691d673 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandler.java @@ -29,7 +29,6 @@ import io.grpc.StatusRuntimeException; import io.micrometer.core.instrument.Counter; -// todo tlongo add tests for metrics public class HttpExceptionHandler implements ExceptionHandlerFunction { private static final Logger LOG = LoggerFactory.getLogger(HttpExceptionHandler.class); @@ -61,7 +60,6 @@ public HttpResponse handleException(final ServiceRequestContext ctx, StatusHolder statusHolder = createStatus(exceptionCause); try { - // todo tlongo why do we need this in the first place? JsonFormat.TypeRegistry typeRegistry = JsonFormat.TypeRegistry.newBuilder() .add(RetryInfo.getDescriptor()) .build(); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java index 8ecc43d7cb..b27416141b 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java @@ -23,7 +23,6 @@ public class HttpService { private static final Logger LOG = LoggerFactory.getLogger(HttpService.class); - // todo tlongo include in config private static final RetryInfoConfig DEFAULT_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(100), Duration.ofMillis(2000)); private final PluginMetrics pluginMetrics; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 50d6685a8a..09287e4f31 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -126,7 +126,7 @@ class OTelTraceSource_HttpServiceTest { private PipelineDescription pipelineDescription; private OTelTraceSource SOURCE; - private static HttpBasicAuthenticationConfig PROVIDED_CONFIG = new HttpBasicAuthenticationConfig("username", "password"); + private static final HttpBasicAuthenticationConfig PROVIDED_CONFIG = new HttpBasicAuthenticationConfig("username", "password"); @BeforeEach @@ -175,6 +175,25 @@ private void configureObjectUnderTest() { SOURCE = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, pipelineDescription); } + @Test + void healthcheck_is_up() { + when(oTelTraceSourceConfig.enableHttpHealthCheck()).thenReturn(true); + SOURCE.start(buffer); + + WebClient.of().execute(RequestHeaders.builder() + .scheme(SessionProtocol.HTTP) + .authority("127.0.0.1:21890") + .method(HttpMethod.HEAD) + .path("/health") + .contentType(MediaType.JSON_UTF_8) + .build()) + .aggregate() + .whenComplete((response, throwable) -> assertThat(response.status(), is(HttpStatus.OK))) + .join(); + + verifyNoInteractions(buffer); + } + @Test void request_fails_because_of_invalid_payload() throws Exception { ExportTraceServiceRequest request = createInvalidExportTraceRequest(); diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandlerTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandlerTest.java new file mode 100644 index 0000000000..7023902e85 --- /dev/null +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpExceptionHandlerTest.java @@ -0,0 +1,97 @@ +package org.opensearch.dataprepper.plugins.source.oteltrace.http; + + +import java.time.Duration; +import java.util.concurrent.TimeoutException; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.HttpRequestExceptionHandler; +import org.opensearch.dataprepper.exceptions.BadRequestException; +import org.opensearch.dataprepper.exceptions.BufferWriteException; +import org.opensearch.dataprepper.exceptions.RequestCancelledException; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.buffer.SizeOverflowException; + +import com.linecorp.armeria.common.HttpRequest; +import com.linecorp.armeria.server.RequestTimeoutException; +import com.linecorp.armeria.server.ServiceRequestContext; + +import io.micrometer.core.instrument.Counter; + +@ExtendWith(MockitoExtension.class) +class HttpExceptionHandlerTest { + HttpExceptionHandler httpExceptionHandler; + + @Mock + private PluginMetrics pluginMetrics; + + @Mock + private ServiceRequestContext requestContext; + + @Mock + private HttpRequest httpRequest; + + @Mock + private Counter requestTimeoutsCounter; + + @Mock + private Counter badRequestsCounter; + + @Mock + private Counter requestsTooLargeCounter; + + @Mock + private Counter internalServerErrorCounter; + @BeforeEach + public void setUp() { + when(pluginMetrics.counter(HttpRequestExceptionHandler.REQUEST_TIMEOUTS)).thenReturn(requestTimeoutsCounter); + when(pluginMetrics.counter(HttpRequestExceptionHandler.BAD_REQUESTS)).thenReturn(badRequestsCounter); + when(pluginMetrics.counter(HttpRequestExceptionHandler.REQUESTS_TOO_LARGE)).thenReturn(requestsTooLargeCounter); + when(pluginMetrics.counter(HttpRequestExceptionHandler.INTERNAL_SERVER_ERROR)).thenReturn(internalServerErrorCounter); + httpExceptionHandler = new HttpExceptionHandler(pluginMetrics, Duration.ofMillis(100), Duration.ofSeconds(2)); + } + + @Test + public void testHandleBadRequestException() { + httpExceptionHandler.handleException(requestContext, httpRequest, new BadRequestException("msg", null)); + verify(badRequestsCounter).increment(); + } + + @Test + public void testHandleTimeoutException() { + httpExceptionHandler.handleException(requestContext, httpRequest, new BufferWriteException(null, new TimeoutException())); + verify(requestTimeoutsCounter, times(1)).increment(); + } + + @Test + public void testHandleArmeriaTimeoutException() { + httpExceptionHandler.handleException(requestContext, httpRequest, RequestTimeoutException.get()); + verify(requestTimeoutsCounter, times(1)).increment(); + } + + @Test + public void testHandleSizeOverflowException() { + httpExceptionHandler.handleException(requestContext, httpRequest, new SizeOverflowException("msg")); + verify(requestsTooLargeCounter).increment(); + } + + @Test + public void testHandleRequestCancelledException() { + httpExceptionHandler.handleException(requestContext, httpRequest, new RequestCancelledException("msg")); + verify(requestTimeoutsCounter, times(1)).increment(); + } + + @Test + public void testHandleInternalServerException() { + httpExceptionHandler.handleException(requestContext, httpRequest, new RuntimeException("msg")); + verify(internalServerErrorCounter, times(1)).increment(); + } +} From a342e1408681dbaaf498eb475e9200393f93f459 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 14 Feb 2025 13:07:17 +0100 Subject: [PATCH 22/30] Remove tests Signed-off-by: Tomas Longo --- .../source/oteltrace/OTelTraceSourceTest.java | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index b941d38e55..03c5403e45 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -607,66 +607,6 @@ void testOptionalHttpAuthServiceNotInPlace() { verify(serverBuilder, never()).decorator(isA(Function.class)); } - @Test - @Disabled - void testOptionalHttpAuthServiOceInPlace() { - final Optional> function = Optional.of(httpService -> httpService); - - final Map settingsMap = new HashMap<>(); - // todo tlongo: Providing this pipeline config to data prepper let's it refuse to boot right away. Why is this test green, then? - // Because the it's the plugin factory that stumbles over the missing plugin settings. But, the plugin factory is - // mocked in this whole test class, always returning a GrpcBasicAuthenticationProvider. IMO this test does not - // service its purpose - settingsMap.put("authentication", new PluginModel("test", null)); - settingsMap.put("unauthenticated_health_check", true); - - settingsMap.put(SSL, false); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - - when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); - - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - - try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - } - - verify(serverBuilder).service(isA(GrpcService.class)); - verify(serverBuilder).decorator(isA(String.class), isA(Function.class)); - } - - @Test - @Disabled // todo tlongo see testOptionalHttpAuthServiOceInPlace on why I disabled this test - void testOptionalHttpAuthServiceInPlaceWithUnauthenticatedDisabled() { - final Optional> function = Optional.of(httpService -> httpService); - - final Map settingsMap = new HashMap<>(); - settingsMap.put("authentication", new PluginModel("test", null)); - settingsMap.put("unauthenticated_health_check", false); - - settingsMap.put(SSL, false); - - testPluginSetting = new PluginSetting(null, settingsMap); - testPluginSetting.setPipelineName("pipeline"); - oTelTraceSourceConfig = OBJECT_MAPPER.convertValue(testPluginSetting.getSettings(), OTelTraceSourceConfig.class); - - when(authenticationProvider.getHttpAuthenticationService()).thenReturn(function); - - final OTelTraceSource source = new OTelTraceSource(oTelTraceSourceConfig, pluginMetrics, pluginFactory, certificateProviderFactory, pipelineDescription); - - try (final MockedStatic armeriaServerMock = Mockito.mockStatic(Server.class)) { - armeriaServerMock.when(Server::builder).thenReturn(serverBuilder); - source.start(buffer); - } - - verify(serverBuilder).service(isA(GrpcService.class)); - verify(serverBuilder).decorator(isA(Function.class)); - } - @Test void testDoubleStart() { // starting server From 6c230f36f8096ba1d00f718a24f8921589696ea3 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 11 Apr 2025 10:29:10 +0200 Subject: [PATCH 23/30] Fix missing imports Signed-off-by: Tomas Longo --- .../dataprepper/integration/trace/EndToEndRawSpanTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index 157de7920f..08bbd0261e 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -21,6 +21,7 @@ import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.resource.v1.Resource; import io.opentelemetry.proto.trace.v1.ResourceSpans; +import io.opentelemetry.proto.trace.v1.ScopeSpans; import io.opentelemetry.proto.trace.v1.Span; import io.opentelemetry.proto.trace.v1.Status; import org.opensearch.action.admin.indices.refresh.RefreshRequest; @@ -99,7 +100,7 @@ public void testPipelineEndToEnd() { final SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); final List> foundSources = getSourcesFromSearchHits(searchResponse.getHits()); - assertThat(expectedDocuments).hasSize(foundSources.size()); + assertThat(foundSources).hasSize(expectedDocuments.size()); assertThatFoundDocumentsContainAllFieldsFromExpectedDocuments(expectedDocuments, foundSources); } ); From fa2e95dfdcd34a6180941cead5d73008a83f3a5b Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Tue, 15 Apr 2025 15:23:28 +0200 Subject: [PATCH 24/30] Remove unused imports Signed-off-by: Tomas Longo --- .../plugins/source/oteltrace/OTelTraceSourceTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java index 03c5403e45..7a923ff0dd 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSourceTest.java @@ -19,7 +19,6 @@ import com.linecorp.armeria.common.MediaType; import com.linecorp.armeria.common.RequestHeaders; import com.linecorp.armeria.common.SessionProtocol; -import com.linecorp.armeria.server.HttpService; import com.linecorp.armeria.server.Server; import com.linecorp.armeria.server.ServerBuilder; import com.linecorp.armeria.server.grpc.GrpcService; @@ -40,7 +39,6 @@ import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -83,7 +81,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.StringJoiner; import java.util.UUID; import java.util.concurrent.CompletableFuture; From 4c694f58571fbadc969acb176d9101f97e46d123 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Wed, 14 May 2025 15:44:19 +0200 Subject: [PATCH 25/30] Fix imports Signed-off-by: Tomas Longo --- .../plugins/source/oteltrace/OTelTraceSource.java | 1 - .../plugins/source/oteltrace/grpc/GrpcService.java | 8 ++++---- .../plugins/source/oteltrace/http/ArmeriaHttpService.java | 3 ++- .../plugins/source/oteltrace/http/HttpService.java | 2 +- .../source/oteltrace/OTelTraceSource_GrpcRequestTest.java | 2 ++ .../oteltrace/OTelTraceSource_UnframedRequestsTest.java | 3 ++- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index 654d26416c..f69fd40b44 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -25,7 +25,6 @@ import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; -import org.opensearch.dataprepper.model.codec.ByteDecoder; import org.opensearch.dataprepper.plugins.otel.codec.OTelTraceDecoder; import org.opensearch.dataprepper.plugins.source.oteltrace.grpc.GrpcService; import org.opensearch.dataprepper.plugins.source.oteltrace.http.HttpService; diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index 5dc9723236..a5f1f59f28 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -14,11 +14,11 @@ import org.opensearch.dataprepper.model.configuration.PluginSetting; import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.health.HealthGrpcService; -import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoOpensearchCodec; +import org.opensearch.dataprepper.plugins.server.HealthGrpcService; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceGrpcService; import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; -import org.opensearch.dataprepper.plugins.source.oteltrace.RetryInfoConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,7 +60,7 @@ public com.linecorp.armeria.server.grpc.GrpcService create(Buffer final OTelTraceGrpcService oTelTraceGrpcService = new OTelTraceGrpcService( (int)(oTelTraceSourceConfig.getRequestTimeoutInMillis() * 0.8), - new OTelProtoCodec.OTelProtoDecoder(), + new OTelProtoOpensearchCodec.OTelProtoDecoder(), buffer, pluginMetrics ); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index a16a301c60..320f74b250 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -14,6 +14,7 @@ import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.trace.Span; import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoCodec; +import org.opensearch.dataprepper.plugins.otel.codec.OTelProtoOpensearchCodec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +52,7 @@ public class ArmeriaHttpService { public ArmeriaHttpService(Buffer> buffer, final PluginMetrics pluginMetrics, final int bufferWriteTimeoutInMillis) { this.buffer = buffer; - this.oTelProtoDecoder = new OTelProtoCodec.OTelProtoDecoder(); + this.oTelProtoDecoder = new OTelProtoOpensearchCodec.OTelProtoDecoder(); this.bufferWriteTimeoutInMillis = bufferWriteTimeoutInMillis; requestsReceivedCounter = pluginMetrics.counter(REQUESTS_RECEIVED); diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java index b27416141b..107010a8b5 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/HttpService.java @@ -13,8 +13,8 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.OTelTraceSourceConfig; -import org.opensearch.dataprepper.plugins.source.oteltrace.RetryInfoConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java index d4ab75ce8c..876e68bb53 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_GrpcRequestTest.java @@ -72,6 +72,7 @@ import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; import com.google.protobuf.ByteString; @@ -111,6 +112,7 @@ class OTelTraceSource_GrpcRequestTest { private static final String TEST_PIPELINE_NAME = "test_pipeline"; private static final RetryInfoConfig TEST_RETRY_INFO = new RetryInfoConfig(Duration.ofMillis(50), Duration.ofMillis(2000)); + @Mock private ServerBuilder serverBuilder; diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java index 1d19507225..56891e4e69 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java @@ -63,7 +63,8 @@ import org.opensearch.dataprepper.plugins.certificate.CertificateProvider; import org.opensearch.dataprepper.plugins.certificate.model.Certificate; import org.opensearch.dataprepper.plugins.codec.CompressionOption; -import org.opensearch.dataprepper.plugins.health.HealthGrpcService; +import org.opensearch.dataprepper.plugins.server.HealthGrpcService; +import org.opensearch.dataprepper.plugins.server.RetryInfoConfig; import org.opensearch.dataprepper.plugins.source.oteltrace.certificate.CertificateProviderFactory; import com.fasterxml.jackson.databind.ObjectMapper; From bfdc669a253cb610885e3546409f9bb5704aebd0 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 23 May 2025 08:36:50 +0200 Subject: [PATCH 26/30] Remove/edit todos Signed-off-by: Tomas Longo --- .../dataprepper/plugins/source/oteltrace/OTelTraceSource.java | 3 ++- .../dataprepper/plugins/source/oteltrace/grpc/GrpcService.java | 3 +-- .../plugins/source/oteltrace/http/ArmeriaHttpService.java | 2 +- .../source/oteltrace/OTelTraceSource_HttpServiceTest.java | 2 +- .../source/oteltrace/OTelTraceSource_UnframedRequestsTest.java | 2 +- examples/trace-analytics-sample-app/docker-compose.yml | 1 + 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java index f69fd40b44..69a1a01cb6 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource.java @@ -89,7 +89,8 @@ public void start(Buffer> buffer) { configureTaskExecutor(serverBuilder); configureGrpcService(serverBuilder, buffer); - // todo tlongo needed until clarified if unframedRequests should survive + + // todo needed until clarified if unframedRequests should survive if (!oTelTraceSourceConfig.enableUnframedRequests()) { configureHttpService(serverBuilder, buffer); } diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java index a5f1f59f28..f90792c80f 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/grpc/GrpcService.java @@ -88,13 +88,12 @@ public com.linecorp.armeria.server.grpc.GrpcService create(Buffer grpcServiceBuilder.addService(new HealthGrpcService()); } - // todo tlongo extract into separate grpc config if (oTelTraceSourceConfig.hasProtoReflectionService()) { LOG.info("Proto reflection service is enabled"); grpcServiceBuilder.addService(ProtoReflectionService.newInstance()); } - // todo tlongo still needed with new http-service? + // todo still needed with new http-service? grpcServiceBuilder.enableUnframedRequests(oTelTraceSourceConfig.enableUnframedRequests()); if (oTelTraceSourceConfig.getAuthentication() != null) { diff --git a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java index 320f74b250..9dc3949a08 100644 --- a/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java +++ b/data-prepper-plugins/otel-trace-source/src/main/java/org/opensearch/dataprepper/plugins/source/oteltrace/http/ArmeriaHttpService.java @@ -61,7 +61,7 @@ public ArmeriaHttpService(Buffer> buffer, final PluginMetrics plu requestProcessDuration = pluginMetrics.timer(REQUEST_PROCESS_DURATION); } - // todo tlongo make path configurable + // todo make path configurable @Post("/opentelemetry.proto.collector.trace.v1.TraceService/Export") @Consumes(value = "application/json") public ExportTraceServiceResponse exportTrace(ExportTraceServiceRequest request) { diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java index 09287e4f31..639117cac6 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_HttpServiceTest.java @@ -379,7 +379,7 @@ private ExportTraceServiceRequest createExportTraceRequest() { private void assertResponseBodyForRetryInformation(final AggregatedHttpResponse response, String expectedDelay) { String body = response.content(StandardCharsets.UTF_8); - // todo tlongo map to numeric value when creating status in exception handler + // todo map to numeric value when creating status in exception handler assertThat(body, hasJsonPath("$.details[0].retryDelay", equalTo(expectedDelay))); } } diff --git a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java index 56891e4e69..88a4a7e550 100644 --- a/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java +++ b/data-prepper-plugins/otel-trace-source/src/test/java/org/opensearch/dataprepper/plugins/source/oteltrace/OTelTraceSource_UnframedRequestsTest.java @@ -97,7 +97,7 @@ import io.opentelemetry.proto.trace.v1.Span; -// todo tlongo check if unframed requests are still needed. If not, remove this whole test class +// todo check if unframed requests are still needed. If not, remove this whole test class @ExtendWith(MockitoExtension.class) class OTelTraceSource_UnframedRequestsTest { // used to configure the path for unframed requests and make sure not to use the same path diff --git a/examples/trace-analytics-sample-app/docker-compose.yml b/examples/trace-analytics-sample-app/docker-compose.yml index d0522ee1ac..085aefff55 100644 --- a/examples/trace-analytics-sample-app/docker-compose.yml +++ b/examples/trace-analytics-sample-app/docker-compose.yml @@ -21,6 +21,7 @@ services: - discovery.type=single-node - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=Prpper123Prepper" ulimits: memlock: soft: -1 From 702a568c34adf266d455f633fb32488211aababf Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 23 May 2025 15:03:41 +0200 Subject: [PATCH 27/30] Remove accidentally added default password Signed-off-by: Tomas Longo --- examples/trace-analytics-sample-app/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/trace-analytics-sample-app/docker-compose.yml b/examples/trace-analytics-sample-app/docker-compose.yml index 085aefff55..d0522ee1ac 100644 --- a/examples/trace-analytics-sample-app/docker-compose.yml +++ b/examples/trace-analytics-sample-app/docker-compose.yml @@ -21,7 +21,6 @@ services: - discovery.type=single-node - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=Prpper123Prepper" ulimits: memlock: soft: -1 From 655abe4583d5ed5c1df9071d162b418aaae57221 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Fri, 20 Jun 2025 10:02:26 +0200 Subject: [PATCH 28/30] Declare assertJ as test lib and reference it from e2e test Signed-off-by: Tomas Longo --- e2e-test/trace/build.gradle | 2 +- settings.gradle | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e-test/trace/build.gradle b/e2e-test/trace/build.gradle index 30c243ff05..c5cf041364 100644 --- a/e2e-test/trace/build.gradle +++ b/e2e-test/trace/build.gradle @@ -207,7 +207,7 @@ dependencies { integrationTestImplementation project(':data-prepper-plugins:aws-plugin-api') integrationTestImplementation project(':data-prepper-plugins:otel-trace-group-processor') integrationTestImplementation testLibs.awaitility - integrationTestImplementation "org.assertj:assertj-core:3.25.3" + integrationTestImplementation testLibs.assertj integrationTestImplementation "io.opentelemetry.proto:opentelemetry-proto:${targetOpenTelemetryVersion}" integrationTestImplementation libs.protobuf.util integrationTestImplementation libs.armeria.core diff --git a/settings.gradle b/settings.gradle index d7dfc2b90a..7729fc0ce1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -80,6 +80,7 @@ dependencyResolutionManagement { version('awaitility', '4.2.0') version('spring', '5.3.28') version('slf4j', '2.0.6') + version('assertj', '3.27.3') library('junit-core', 'org.junit.jupiter', 'junit-jupiter').versionRef('junit') library('junit-params', 'org.junit.jupiter', 'junit-jupiter-params').versionRef('junit') library('junit-engine', 'org.junit.jupiter', 'junit-jupiter-engine').versionRef('junit') @@ -93,6 +94,7 @@ dependencyResolutionManagement { library('awaitility', 'org.awaitility', 'awaitility').versionRef('awaitility') library('spring-test', 'org.springframework', 'spring-test').versionRef('spring') library('slf4j-simple', 'org.slf4j', 'slf4j-simple').versionRef('slf4j') + library('assertj', 'org.assertj', 'assertj-core').versionRef('assertj') } } } From fe1e86adb09d2ae9c964b1d3833a2668767d30b4 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Mon, 27 Oct 2025 11:57:15 +0100 Subject: [PATCH 29/30] Fix merge error Signed-off-by: Tomas Longo --- .../trace/EndToEndRawSpanTest.java | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index acb9c56060..08bbd0261e 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -162,42 +162,6 @@ private List> sendTracesToOpenSearchAndReturnExpectedDocumen return getExpectedDocuments( exportTraceServiceRequestTrace1BatchWithRoot, exportTraceServiceRequestTrace1BatchNoRoot, exportTraceServiceRequestTrace2BatchWithRoot, exportTraceServiceRequestTrace2BatchNoRoot); - final ConnectionConfiguration.Builder builder = new ConnectionConfiguration.Builder( - Collections.singletonList("https://127.0.0.1:9200")); - builder.withUsername("admin"); - builder.withPassword("admin"); - builder.withInsecure(true); - final RestHighLevelClient restHighLevelClient = builder.build().createClient(null); - // Wait for data to flow through pipeline and be indexed by ES - await().atLeast(3, TimeUnit.SECONDS).atMost(20, TimeUnit.SECONDS).untilAsserted( - () -> { - refreshIndices(restHighLevelClient); - final SearchRequest searchRequest = new SearchRequest(INDEX_NAME); - searchRequest.source( - SearchSourceBuilder.searchSource() - .size(100) - .fetchField(TraceGroup.TRACE_GROUP_STATUS_CODE_FIELD) - .fetchField(TraceGroup.TRACE_GROUP_END_TIME_FIELD, "strict_date_time") - .fetchField(TraceGroup.TRACE_GROUP_DURATION_IN_NANOS_FIELD) - ); - final SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - final List> foundSources = getSourcesFromSearchHits(searchResponse.getHits()); - assertEquals(expectedDocuments.size(), foundSources.size()); - /** - * Our raw trace prepper add more fields than the actual sent object. These are defaults from the proto. - * So assertion is done if all the expected fields exists. - * - * TODO: Can we do better? - * - */ - expectedDocuments.forEach(expectedDoc -> { - assertTrue(foundSources.stream() - .filter(i -> i.get("spanId").equals(expectedDoc.get("spanId"))) - .findFirst().get() - .entrySet().containsAll(expectedDoc.entrySet())); - }); - } - ); } private void refreshIndices(final RestHighLevelClient restHighLevelClient) throws IOException { From 39c81425c03d20aa914481e7a83e9629292ffe46 Mon Sep 17 00:00:00 2001 From: Tomas Longo Date: Wed, 29 Oct 2025 10:48:25 +0100 Subject: [PATCH 30/30] Use insecure ssl connection in test Signed-off-by: Tomas Longo --- .../dataprepper/integration/trace/EndToEndRawSpanTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java index 08bbd0261e..571f5f45af 100644 --- a/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java +++ b/e2e-test/trace/src/integrationTest/java/org/opensearch/dataprepper/integration/trace/EndToEndRawSpanTest.java @@ -136,6 +136,7 @@ private RestHighLevelClient createRestClientForSearch() { return new ConnectionConfiguration.Builder(Collections.singletonList("https://127.0.0.1:9200")) .withUsername("admin") .withPassword("admin") + .withInsecure(true) .build().createClient(null); }