feat(infra): add OpenTelemetry distributed tracing across all services (STA-221) (#234)

Add end-to-end distributed tracing using Micrometer Tracing + OpenTelemetry:

- TracingConfig: conditional toggle via app.tracing.enabled (default true)
- KafkaTracingConfig: BeanPostProcessor enables observation on all
  KafkaTemplate and KafkaListenerContainerFactory instances for trace
  context propagation through Kafka message headers (W3C traceparent)
- Jaeger all-in-one added to docker-compose.dev.yml (UI on :16686,
  OTLP HTTP on :4318, OTLP gRPC on :4317)
- OTEL collector reference config in infra/local/otel/ for production
- 8 new tests covering TracingConfig and KafkaTracingConfig behavior

Dependencies already in place (from convention plugin):
  - micrometer-tracing-bridge-otel
  - opentelemetry-exporter-otlp

All 10 services already configured with:
  - management.tracing.sampling.probability: 1.0
  - management.otlp.tracing.endpoint: http://localhost:4318/v1/traces

Closes STA-221

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Puneethkumarck

docker-compose.dev.yml

@@ -220,6 +220,24 @@ services:
       - ./infra/local/wiremock/__files:/home/wiremock/__files
     command: --verbose --global-response-templating
 
+  # ─────────────────────────────────────────────
+  # Jaeger — distributed trace visualization (STA-221)
+  # ─────────────────────────────────────────────
+  jaeger:
+    image: jaegertracing/all-in-one:1.67
+    container_name: sp-jaeger
+    ports:
+      - "16686:16686"  # Jaeger UI
+      - "4317:4317"    # OTLP gRPC receiver
+      - "4318:4318"    # OTLP HTTP receiver
+    environment:
+      COLLECTOR_OTLP_ENABLED: "true"
+    healthcheck:
+      test: ["CMD-SHELL", "wget --spider -q http://localhost:16686/ || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
 volumes:
   pgdata:
   tsdata:

infra/local/otel/otel-collector-config.yml

@@ -0,0 +1,42 @@
+# OpenTelemetry Collector configuration — optional for production deployments.
+#
+# For LOCAL DEV, services export OTLP directly to Jaeger (which has built-in OTLP support).
+# This config is provided as a reference for production where a dedicated collector
+# sits between services and the tracing backend (Jaeger, Tempo, Datadog, etc.).
+#
+# To use in docker-compose, add an otel-collector service:
+#   otel-collector:
+#     image: otel/opentelemetry-collector-contrib:0.100.0
+#     ports:
+#       - "4317:4317"
+#       - "4318:4318"
+#     volumes:
+#       - ./infra/local/otel/otel-collector-config.yml:/etc/otelcol-contrib/config.yaml
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 512
+
+exporters:
+  otlp/jaeger:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+  debug:
+    verbosity: basic
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/jaeger, debug]

platform-infra/build.gradle.kts

@@ -21,6 +21,7 @@ dependencies {
     annotationProcessor("org.projectlombok:lombok")
 
     testImplementation("org.springframework.boot:spring-boot-starter-test")
+    testImplementation("org.springframework.boot:spring-boot-starter-kafka")
     testImplementation("org.springframework.cloud:spring-cloud-starter-openfeign")
     testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 }

platform-infra/src/main/java/com/stablecoin/payments/platform/infrastructure/tracing/KafkaTracingConfig.java

@@ -0,0 +1,55 @@
+package com.stablecoin.payments.platform.infrastructure.tracing;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.config.BeanPostProcessor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
+import org.springframework.kafka.core.KafkaTemplate;
+
+/**
+ * Enables Micrometer Observation on Kafka producers and consumers so that trace context
+ * (W3C {@code traceparent}) is automatically propagated through Kafka message headers.
+ *
+ * <p>This ensures end-to-end trace visibility across the event-driven pipeline:
+ * S1 (Orchestrator) &rarr; Kafka &rarr; S2 (Compliance) &rarr; Kafka &rarr; S6 (FX) etc.
+ *
+ * <p>Uses a {@link BeanPostProcessor} to enable observation on all {@link KafkaTemplate}
+ * and {@link ConcurrentKafkaListenerContainerFactory} instances, including those created
+ * manually by individual services (not just auto-configured beans).
+ *
+ * <p>Activated only when both tracing and Kafka are on the classpath.
+ */
+@Slf4j
+@Configuration
+@ConditionalOnProperty(name = "app.tracing.enabled", havingValue = "true", matchIfMissing = true)
+@ConditionalOnClass(KafkaTemplate.class)
+public class KafkaTracingConfig {
+
+    @Bean
+    static KafkaObservationBeanPostProcessor kafkaObservationBeanPostProcessor() {
+        return new KafkaObservationBeanPostProcessor();
+    }
+
+    /**
+     * Post-processor that enables observation on all Kafka producer templates and
+     * consumer container factories discovered in the application context.
+     */
+    static class KafkaObservationBeanPostProcessor implements BeanPostProcessor {
+
+        @Override
+        public Object postProcessAfterInitialization(Object bean, String beanName) {
+            if (bean instanceof KafkaTemplate<?, ?> template) {
+                template.setObservationEnabled(true);
+                log.debug("Enabled observation on KafkaTemplate bean '{}'", beanName);
+            }
+            if (bean instanceof ConcurrentKafkaListenerContainerFactory<?, ?> factory) {
+                factory.getContainerProperties().setObservationEnabled(true);
+                log.debug("Enabled observation on KafkaListenerContainerFactory bean '{}'", beanName);
+            }
+            return bean;
+        }
+    }
+}

platform-infra/src/main/java/com/stablecoin/payments/platform/infrastructure/tracing/TracingConfig.java

@@ -0,0 +1,37 @@
+package com.stablecoin.payments.platform.infrastructure.tracing;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Enables distributed tracing across all services via Spring Boot auto-configuration.
+ *
+ * <p>When {@code app.tracing.enabled=true} (default), Spring Boot auto-configures:
+ * <ul>
+ *   <li>Micrometer Tracing bridge to OpenTelemetry ({@code micrometer-tracing-bridge-otel})</li>
+ *   <li>OTLP HTTP span exporter to the collector configured via
+ *       {@code management.otlp.tracing.endpoint}</li>
+ *   <li>W3C {@code traceparent} header propagation on Feign/RestClient calls</li>
+ *   <li>Trace context propagation through Kafka message headers when observation is enabled</li>
+ *   <li>MDC population with {@code traceId} and {@code spanId} for structured logging</li>
+ * </ul>
+ *
+ * <p>Set {@code app.tracing.enabled=false} in tests or environments without a collector.
+ */
+@Configuration
+@ConditionalOnProperty(name = "app.tracing.enabled", havingValue = "true", matchIfMissing = true)
+public class TracingConfig {
+
+    // Spring Boot auto-configures all tracing beans (OtlpHttpSpanExporter, TracerProvider,
+    // ContextPropagators) when micrometer-tracing-bridge-otel and opentelemetry-exporter-otlp
+    // are on the classpath. No manual bean definitions needed.
+    //
+    // Key application properties driving behavior:
+    //   management.tracing.sampling.probability  — sampling rate (1.0 = 100% in dev)
+    //   management.otlp.tracing.endpoint         — OTLP collector HTTP endpoint
+    //
+    // This @Configuration class exists to:
+    //   1. Provide a single toggle (app.tracing.enabled) to disable all tracing
+    //   2. Document the tracing architecture for developers
+    //   3. Serve as an extension point for custom SpanProcessor/SpanExporter beans
+}

platform-infra/src/test/java/com/stablecoin/payments/platform/infrastructure/tracing/KafkaTracingConfigTest.java

@@ -0,0 +1,100 @@
+package com.stablecoin.payments.platform.infrastructure.tracing;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.autoconfigure.AutoConfigurations;
+import org.springframework.boot.kafka.autoconfigure.KafkaAutoConfiguration;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+import org.springframework.kafka.core.KafkaTemplate;
+
+import java.lang.reflect.Field;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class KafkaTracingConfigTest {
+
+    private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
+            .withConfiguration(AutoConfigurations.of(
+                    KafkaAutoConfiguration.class,
+                    KafkaTracingConfig.class))
+            .withPropertyValues("spring.kafka.bootstrap-servers=localhost:9092");
+
+    @Test
+    @DisplayName("should enable observation on KafkaTemplate when tracing is enabled")
+    void shouldEnableObservationOnKafkaTemplate() {
+        contextRunner
+                .withPropertyValues("app.tracing.enabled=true")
+                .run(context -> {
+                    assertThat(context).hasSingleBean(KafkaTemplate.class);
+                    var template = context.getBean(KafkaTemplate.class);
+                    assertThat(isObservationEnabled(template)).isTrue();
+                });
+    }
+
+    @Test
+    @DisplayName("should enable observation by default when app.tracing.enabled is absent")
+    void shouldEnableObservationByDefault() {
+        contextRunner
+                .run(context -> {
+                    assertThat(context).hasSingleBean(KafkaTemplate.class);
+                    var template = context.getBean(KafkaTemplate.class);
+                    assertThat(isObservationEnabled(template)).isTrue();
+                });
+    }
+
+    @Test
+    @DisplayName("should not enable observation when app.tracing.enabled is false")
+    void shouldNotEnableObservationWhenTracingDisabled() {
+        new ApplicationContextRunner()
+                .withConfiguration(AutoConfigurations.of(
+                        KafkaAutoConfiguration.class,
+                        KafkaTracingConfig.class))
+                .withPropertyValues(
+                        "spring.kafka.bootstrap-servers=localhost:9092",
+                        "app.tracing.enabled=false")
+                .run(context -> {
+                    assertThat(context).doesNotHaveBean(KafkaTracingConfig.class);
+                    assertThat(context).hasSingleBean(KafkaTemplate.class);
+                    var template = context.getBean(KafkaTemplate.class);
+                    assertThat(isObservationEnabled(template)).isFalse();
+                });
+    }
+
+    @Test
+    @DisplayName("should register BeanPostProcessor when tracing is enabled")
+    void shouldRegisterBeanPostProcessor() {
+        contextRunner
+                .run(context ->
+                        assertThat(context).hasSingleBean(
+                                KafkaTracingConfig.KafkaObservationBeanPostProcessor.class));
+    }
+
+    @Test
+    @DisplayName("should not register BeanPostProcessor when tracing is disabled")
+    void shouldNotRegisterBeanPostProcessorWhenDisabled() {
+        new ApplicationContextRunner()
+                .withConfiguration(AutoConfigurations.of(
+                        KafkaAutoConfiguration.class,
+                        KafkaTracingConfig.class))
+                .withPropertyValues(
+                        "spring.kafka.bootstrap-servers=localhost:9092",
+                        "app.tracing.enabled=false")
+                .run(context ->
+                        assertThat(context).doesNotHaveBean(
+                                KafkaTracingConfig.KafkaObservationBeanPostProcessor.class));
+    }
+
+    /**
+     * Reads the private {@code observationEnabled} field via reflection since
+     * {@link KafkaTemplate} exposes a setter but no getter for this flag.
+     */
+    private static boolean isObservationEnabled(KafkaTemplate<?, ?> template) {
+        try {
+            Field field = KafkaTemplate.class.getDeclaredField("observationEnabled");
+            field.setAccessible(true);
+            return (boolean) field.get(template);
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            throw new RuntimeException("Failed to read observationEnabled field", e);
+        }
+    }
+}

platform-infra/src/test/java/com/stablecoin/payments/platform/infrastructure/tracing/TracingConfigTest.java

@@ -0,0 +1,37 @@
+package com.stablecoin.payments.platform.infrastructure.tracing;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.autoconfigure.AutoConfigurations;
+import org.springframework.boot.test.context.runner.ApplicationContextRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class TracingConfigTest {
+
+    private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
+            .withConfiguration(AutoConfigurations.of(TracingConfig.class));
+
+    @Test
+    @DisplayName("should register TracingConfig when app.tracing.enabled is true")
+    void shouldRegisterTracingConfigWhenEnabled() {
+        contextRunner
+                .withPropertyValues("app.tracing.enabled=true")
+                .run(context -> assertThat(context).hasSingleBean(TracingConfig.class));
+    }
+
+    @Test
+    @DisplayName("should register TracingConfig when app.tracing.enabled property is absent (default)")
+    void shouldRegisterTracingConfigWhenPropertyAbsent() {
+        contextRunner
+                .run(context -> assertThat(context).hasSingleBean(TracingConfig.class));
+    }
+
+    @Test
+    @DisplayName("should not register TracingConfig when app.tracing.enabled is false")
+    void shouldNotRegisterTracingConfigWhenDisabled() {
+        contextRunner
+                .withPropertyValues("app.tracing.enabled=false")
+                .run(context -> assertThat(context).doesNotHaveBean(TracingConfig.class));
+    }
+}