feat(observability): propagate W3C trace context across nico-api's network boundaries (#2700)

Implements W3C Trace Context propagation for nico-api so a request
traced upstream keeps one trace as
it passes through nico-api and into the services it calls.

- **Propagator:** install the standard W3C `TraceContextPropagator` at
startup (the OTel default is no-op).
- **Ingress (REST + gRPC):** one shared per-request layer extracts the
inbound `traceparent`/`tracestate`
and parents the request span to it; no valid inbound context → fresh
root.
- **Egress:** gRPC clients are wrapped in a tower `TraceInjectService`;
reqwest clients go through
`reqwest-tracing`'s middleware; the two OAuth2 token providers inject
manually (the `oauth2` crate
  builds their requests).
- **Enable flag unchanged:** the local `tracing-enabled` switch stays
the master control — an inbound
  `sampled` flag can't force local recording.

Per-client details and how to add a new client are in
`docs/observability/tracing.md`.

## Related issues
#2438

## Type of Change
- [x] **Add** - New feature or capability
- [ ] **Change** - Changes in existing functionality
- [ ] **Fix** - Bug fixes
- [ ] **Remove** - Removed features or deprecated functionality
- [ ] **Internal** - Internal changes (refactoring, tests, docs, etc.)

## Breaking Changes
- [ ] **This PR contains breaking changes**

## Testing
- [x] Unit tests added/updated
- [ ] Integration tests added/updated
- [ ] Manual testing performed
- [ ] No testing required (docs, internal refactor, etc.)

## Additional Notes
- **Scope is the nico-api process.** `bmc-proxy` and `hardware-health`
are intentionally **not** instrumented:
they build no span exporter, so injecting there would be dead code.
`docs/observability/tracing.md` §1.7
  documents how to add propagation to a new client if that changes.
- New crate `trace-propagation` holds the shared glue + tower
middleware.
- **`tracing.md` marker references are a doc-staleness fix, not part of
this feature.** The doc still
described the sampler's old `code.namespace` marker; #2268 replaced it
with `carbide.trace_root`
without updating the doc. The `code.namespace`→`carbide.trace_root` doc
edits just correct that.
(The doc's `ParentBased` mention is also updated, since this PR unwraps
the sampler.)
- Dependencies added: `opentelemetry-http` 0.31, `reqwest-middleware`
0.5, `reqwest-tracing` 0.7.

---------

Signed-off-by: Adnan Dosani <258082943+adnandnv@users.noreply.github.com>
Co-authored-by: Adnan Dosani <258082943+adnandnv@users.noreply.github.com>

Author: adnandnv

Cargo.lock

@@ -52,6 +52,7 @@ sqlx-macros-core = { version = "0.9" }
 
 opentelemetry = { version = "0.31.0", features = ["logs"] }
 opentelemetry_sdk = { version = "0.31.0", features = ["logs", "rt-tokio"] }
+opentelemetry-http = "0.31.0"
 opentelemetry-otlp = { version = "0.31.1", features = ["metrics"] }
 opentelemetry-prometheus = "0.31.0"
 opentelemetry-semantic-conventions = "0.31.0"
@@ -194,6 +195,10 @@ ratatui = "0.30.0"
 rcgen = "0.14.7"
 regex = "1.11.2"
 reqwest = { default-features = false, version = "0.13.3" }
+reqwest-middleware = "0.5"
+reqwest-tracing = { version = "0.7", default-features = false, features = [
+  "opentelemetry_0_31",
+] }
 resolv-conf = "0.7.0"
 ringbuf = "0.5.0"
 rsa = "0.9.9"

Cargo.toml

@@ -77,6 +77,7 @@ nras = { path = "../nras" }
 spancounter = { path = "../spancounter" }
 state-controller = { path = "../state-controller" }
 sqlx-query-tracing = { path = "../sqlx-query-tracing" }
+trace-propagation = { path = "../trace-propagation" }
 
 # External dependencies. PLEASE KEEP ALPHABETIZED ORDER.
 arc-swap = { workspace = true }
@@ -133,6 +134,8 @@ reqwest = { workspace = true, default-features = false, features = [
   "rustls",
   "stream",
 ] }
+reqwest-middleware = { workspace = true }
+reqwest-tracing = { workspace = true }
 rsa = { workspace = true }
 rumqttc = { workspace = true }
 rustls = { workspace = true }

crates/api-core/Cargo.toml

@@ -393,7 +393,7 @@ pub(crate) async fn create_client(
         rpc::forge::BmcMetaDataGetResponse,
         http::Uri,
         HeaderMap,
-        reqwest::Client,
+        reqwest_middleware::ClientWithMiddleware,
     ),
     CarbideError,
 > {
@@ -450,15 +450,20 @@ pub(crate) async fn create_client(
             .connect_timeout(std::time::Duration::from_secs(5)) // Limit connections to 5 seconds
             .timeout(std::time::Duration::from_secs(60)); // Limit the overall request to 60 seconds
 
-        match builder.build() {
+        let client = match builder.build() {
             Ok(client) => client,
             Err(err) => {
                 tracing::error!(%err, "build_http_client");
                 return Err(CarbideError::internal(format!(
                     "Http building failed: {err}"
                 )));
             }
-        }
+        };
+        // The `reqwest-tracing` middleware injects the current span's W3C trace context into every
+        // outgoing request (#2438).
+        reqwest_middleware::ClientBuilder::new(client)
+            .with(reqwest_tracing::TracingMiddleware::default())
+            .build()
     };
     Ok((metadata, new_uri, headers, http_client))
 }
@@ -558,15 +563,15 @@ impl TestBehavior {
     }
 }
 
-// Subset of the data we care about from reqwest::Error, so that we can mock it (we can't build our
-// own reqwest::Error as its constructors are all private.)
+// Subset of the data we care about from the HTTP error, so that we can mock it (we can't build our
+// own reqwest error as its constructors are all private.)
 pub struct RequestErrorInfo {
     pub status_code: Option<http::status::StatusCode>,
     pub description: String,
 }
 
-impl From<reqwest::Error> for RequestErrorInfo {
-    fn from(e: reqwest::Error) -> Self {
+impl From<reqwest_middleware::Error> for RequestErrorInfo {
+    fn from(e: reqwest_middleware::Error) -> Self {
         Self {
             status_code: e.status(),
             description: e.to_string(),

crates/api-core/src/handlers/redfish.rs

@@ -163,6 +163,13 @@ where
                 tenant.organization_id = tracing::field::Empty,
             );
 
+            // Continue any distributed trace the caller started: make the inbound W3C trace
+            // context (`traceparent`/`tracestate`) the parent of this request span. The shared
+            // tower layer serves both the gRPC and REST listeners, so this single extractor
+            // covers both ingress paths. No-op when there is no valid inbound context, leaving
+            // the request span a fresh root (issue #2438).
+            trace_propagation::set_span_parent_from_headers(&request_span, request.headers());
+
             // Try to extract the gRPC service and method from the URI
             let mut grpc_method: Option<String> = None;
             let mut grpc_service: Option<String> = None;