feat: add OTLP export into tracing setup with context propagation helpers (#3385)

Signed-off-by: adarsh0728 <gooneriitk@gmail.com>
Signed-off-by: Vigith Maurice <vigith@gmail.com>
Co-authored-by: Vigith Maurice <vigith@gmail.com>

Author: adarsh0728

rust/Cargo.lock

@@ -83,6 +83,10 @@ tokio-stream = "0.1.17"
 tokio-util = { version = "0.7", features = ["rt"] }
 bytes = "1.11.1"
 tracing = "0.1.41"
+tracing-opentelemetry = "0.32"
+opentelemetry = "0.31"
+opentelemetry_sdk = "0.31"
+opentelemetry-otlp = { version = "0.31", features = ["grpc-tonic"] }
 async-nats = "0.44.2"
 thiserror = "2.0.12"
 axum = { version = "0.8.4", default-features = false, features = [

rust/Cargo.toml

@@ -19,6 +19,9 @@ tokio.workspace = true
 tokio-stream.workspace = true
 tokio-util.workspace = true
 tracing.workspace = true
+tracing-opentelemetry.workspace = true
+opentelemetry = { workspace = true }
+opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
 numaflow-pulsar.workspace = true
 numaflow-monitor.workspace = true
 numaflow-nats.workspace = true

rust/numaflow-core/Cargo.toml

@@ -11,6 +11,11 @@ pub(crate) mod create_components;
 /// Shared methods for forwarding messages.
 pub(crate) mod forward;
 
+/// OpenTelemetry propagation helpers for distributed tracing.
+/// Functions in this module are consumed by span creation code in source, mapper, and sinker.
+#[allow(dead_code)]
+pub(crate) mod otel;
+
 /// Test server framework: helpers for spinning up numaflow SDK gRPC servers in tests.
 #[cfg(test)]
 pub(crate) mod test_utils;