From be18c4394d7572f0a1c12adc20db04525d4dd153 Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 22 Jun 2026 13:07:53 -0400 Subject: [PATCH 1/5] fix(trace-stats): add grpc_method to aggregation key Spans with different gRPC methods were previously merged into the same stats group (only the first span's method was kept). Adding grpc_method to FixedAggregationKey ensures each method gets a separate bucket. The OtlpExactGroup.grpc_method field is now sourced from the key rather than a GroupedStats sidecar. The agent /v0.6/stats protobuf wire format is unchanged (no grpc_method field in ClientGroupedStats). SHM_VERSION bumped to 2 because FixedAggregationKey is #[repr(C)] and the new field changes the layout; mismatched sidecar/worker pairs will safely fail with a version-mismatch error. Co-Authored-By: Claude Sonnet 4.6 --- datadog-ipc/src/shm_stats.rs | 3 +- .../src/span_concentrator/aggregation.rs | 29 +++++++++---------- .../src/span_concentrator/mod.rs | 3 +- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/datadog-ipc/src/shm_stats.rs b/datadog-ipc/src/shm_stats.rs index 44941ec667..82c69c9d26 100644 --- a/datadog-ipc/src/shm_stats.rs +++ b/datadog-ipc/src/shm_stats.rs @@ -60,7 +60,7 @@ use libdd_trace_stats::span_concentrator::{FixedAggregationKey, FlushableConcent use crate::platform::{FileBackedHandle, MappedMem, NamedShmHandle}; -const SHM_VERSION: u32 = 1; +const SHM_VERSION: u32 = 2; /// Maximum peer-tag (key, value) pairs per aggregation slot. pub const MAX_PEER_TAGS: usize = 16; @@ -854,6 +854,7 @@ mod tests { http_method: "GET", http_endpoint: "/", service_source: "", + grpc_method: "", http_status_code: 200, is_synthetics_request: false, is_trace_root: true, diff --git a/libdd-trace-stats/src/span_concentrator/aggregation.rs b/libdd-trace-stats/src/span_concentrator/aggregation.rs index cc5f82bdab..21dce3bb5c 100644 --- a/libdd-trace-stats/src/span_concentrator/aggregation.rs +++ b/libdd-trace-stats/src/span_concentrator/aggregation.rs @@ -45,6 +45,7 @@ pub struct FixedAggregationKey { pub http_method: T, pub http_endpoint: T, pub service_source: T, + pub grpc_method: T, pub http_status_code: u32, pub grpc_status_code: Option, pub is_synthetics_request: bool, @@ -69,6 +70,7 @@ impl FixedAggregationKey { http_method: f(self.http_method.borrow()), http_endpoint: f(self.http_endpoint.borrow()), service_source: f(self.service_source.borrow()), + grpc_method: f(self.grpc_method.borrow()), http_status_code: self.http_status_code, grpc_status_code: self.grpc_status_code, is_synthetics_request: self.is_synthetics_request, @@ -153,7 +155,7 @@ fn get_grpc_status_code<'a>(span: &'a impl StatSpan<'a>) -> Option { None } -pub(super) fn get_grpc_method<'a>(span: &'a impl StatSpan<'a>) -> &'a str { +fn get_grpc_method<'a>(span: &'a impl StatSpan<'a>) -> &'a str { for key in GRPC_METHOD_FIELD { if let Some(val) = span.get_meta(key) { if !val.is_empty() { @@ -262,6 +264,7 @@ impl<'a> BorrowedAggregationKey<'a> { }; let grpc_status_code = get_grpc_status_code(span); + let grpc_method = get_grpc_method(span); let service_source = span.get_meta(TAG_SVC_SRC).unwrap_or_default(); @@ -275,6 +278,7 @@ impl<'a> BorrowedAggregationKey<'a> { http_method, http_endpoint, service_source, + grpc_method, http_status_code: status_code, grpc_status_code, is_synthetics_request: span @@ -299,6 +303,7 @@ impl From for OwnedAggregationKey { http_method: value.http_method, http_endpoint: value.http_endpoint, service_source: value.service_source, + grpc_method: String::new(), http_status_code: value.http_status_code, grpc_status_code: value.grpc_status_code.parse().ok(), is_synthetics_request: value.synthetics, @@ -344,9 +349,6 @@ pub(super) struct GroupedStats { error_duration: u64, error_min: u64, error_max: u64, - // gRPC method for OTLP export only; not part of the aggregation key so agent stats are - // unaffected. - pub(super) grpc_method: String, } impl GroupedStats { @@ -390,9 +392,8 @@ pub struct OtlpExactCell { } /// Exact OK/ERROR cells for one aggregation group, in the same order as the `stats` vector -/// of the accompanying [`pb::ClientStatsBucket`]. `grpc_method` is the group's gRPC method (DD -/// schema `grpc.method.name`) carried out-of-band so it does not appear in the agent stats -/// protobuf wire format. +/// of the accompanying [`pb::ClientStatsBucket`]. `grpc_method` mirrors the aggregation key +/// field; it is not in the agent stats protobuf so it is surfaced here for OTLP export. #[derive(Debug, Clone, Default)] pub struct OtlpExactGroup { pub ok: OtlpExactCell, @@ -433,14 +434,10 @@ impl StatsBucket { duration: i64, is_error: bool, is_top_level: bool, - grpc_method: &str, ) { self.data .entry_ref(&key) - .or_insert_with(|| GroupedStats { - grpc_method: grpc_method.to_owned(), - ..Default::default() - }) + .or_default() .insert(duration, is_error, is_top_level); } @@ -455,9 +452,9 @@ impl StatsBucket { pub(super) fn flush_with_otlp_exact(self, bucket_duration: u64) -> OtlpStatsBucket { let mut stats = Vec::with_capacity(self.data.len()); let mut exact = Vec::with_capacity(self.data.len()); - for (k, mut g) in self.data { - let grpc_method = std::mem::take(&mut g.grpc_method); + for (k, g) in self.data { exact.push(OtlpExactGroup { + grpc_method: k.fixed.grpc_method.clone(), ok: OtlpExactCell { count: g.hits.saturating_sub(g.errors), duration_ns: g.ok_duration, @@ -470,7 +467,6 @@ impl StatsBucket { min_ns: g.error_min, max_ns: g.error_max, }, - grpc_method, }); stats.push(encode_grouped_stats(k, g)); } @@ -834,13 +830,14 @@ mod tests { } .into_key(), ), - // grpc.method.name is carried in GroupedStats (for OTLP), not in the aggregation key. + // grpc.method.name is part of the aggregation key. ( SpanBytes { meta: vec![("grpc.method.name".into(), "/pkg.Svc/Method".into())].into(), ..Default::default() }, FixedAggregationKey { + grpc_method: "/pkg.Svc/Method".into(), is_trace_root: true, ..Default::default() } diff --git a/libdd-trace-stats/src/span_concentrator/mod.rs b/libdd-trace-stats/src/span_concentrator/mod.rs index 636e87744d..fb0f4fc4d6 100644 --- a/libdd-trace-stats/src/span_concentrator/mod.rs +++ b/libdd-trace-stats/src/span_concentrator/mod.rs @@ -9,7 +9,7 @@ use libdd_trace_protobuf::pb; use aggregation::StatsBucket; mod aggregation; -use aggregation::{get_grpc_method, BorrowedAggregationKey}; +use aggregation::BorrowedAggregationKey; pub use aggregation::{FixedAggregationKey, OtlpExactCell, OtlpExactGroup, OtlpStatsBucket}; pub mod stat_span; @@ -184,7 +184,6 @@ impl SpanConcentrator { span.duration(), span.is_error(), span.has_top_level(), - get_grpc_method(span), ); } From eb3cfc788fd35068dbd8f2034fd1f1122d385853 Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 22 Jun 2026 13:51:22 -0400 Subject: [PATCH 2/5] ci: trigger fresh workflow run From 75dd3a3020a2b382ec181d7bf3936154aba3135c Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 22 Jun 2026 14:21:19 -0400 Subject: [PATCH 3/5] fix(trace-stats): move grpc_method to end of FixedAggregationKey Bincode encodes struct fields positionally. Inserting grpc_method before http_status_code shifted all subsequent field positions, breaking IPC fallback decoding (OwnedShmSpanInput) between mismatched worker/sidecar versions. Moving it to the end of the struct preserves all existing field positions, so old-format IPC messages are decoded correctly up to the grpc_method field; the decode then fails with EOF rather than silently misinterpreting existing fields. Co-Authored-By: Claude Sonnet 4.6 --- libdd-trace-stats/src/span_concentrator/aggregation.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libdd-trace-stats/src/span_concentrator/aggregation.rs b/libdd-trace-stats/src/span_concentrator/aggregation.rs index 21dce3bb5c..b4a031de15 100644 --- a/libdd-trace-stats/src/span_concentrator/aggregation.rs +++ b/libdd-trace-stats/src/span_concentrator/aggregation.rs @@ -45,11 +45,12 @@ pub struct FixedAggregationKey { pub http_method: T, pub http_endpoint: T, pub service_source: T, - pub grpc_method: T, pub http_status_code: u32, pub grpc_status_code: Option, pub is_synthetics_request: bool, pub is_trace_root: bool, + // Appended last to preserve bincode field-order compatibility with existing IPC messages. + pub grpc_method: T, } impl FixedAggregationKey { @@ -70,11 +71,11 @@ impl FixedAggregationKey { http_method: f(self.http_method.borrow()), http_endpoint: f(self.http_endpoint.borrow()), service_source: f(self.service_source.borrow()), - grpc_method: f(self.grpc_method.borrow()), http_status_code: self.http_status_code, grpc_status_code: self.grpc_status_code, is_synthetics_request: self.is_synthetics_request, is_trace_root: self.is_trace_root, + grpc_method: f(self.grpc_method.borrow()), } } } From 6aa14631e5ed9b2d6b3aa80708dbf423c67c3924 Mon Sep 17 00:00:00 2001 From: Munir Date: Thu, 25 Jun 2026 15:10:53 -0400 Subject: [PATCH 4/5] style(trace-stats): remove stale comments Co-Authored-By: Claude Sonnet 4.6 --- libdd-trace-stats/src/span_concentrator/aggregation.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/libdd-trace-stats/src/span_concentrator/aggregation.rs b/libdd-trace-stats/src/span_concentrator/aggregation.rs index b4a031de15..c2df29ebd2 100644 --- a/libdd-trace-stats/src/span_concentrator/aggregation.rs +++ b/libdd-trace-stats/src/span_concentrator/aggregation.rs @@ -49,7 +49,6 @@ pub struct FixedAggregationKey { pub grpc_status_code: Option, pub is_synthetics_request: bool, pub is_trace_root: bool, - // Appended last to preserve bincode field-order compatibility with existing IPC messages. pub grpc_method: T, } @@ -831,7 +830,6 @@ mod tests { } .into_key(), ), - // grpc.method.name is part of the aggregation key. ( SpanBytes { meta: vec![("grpc.method.name".into(), "/pkg.Svc/Method".into())].into(), From 9ad612d4036b911b79f10bf81c3845d67bce7034 Mon Sep 17 00:00:00 2001 From: Munir Date: Thu, 25 Jun 2026 16:36:54 -0400 Subject: [PATCH 5/5] fix(trace-stats)!: remove grpc_method from aggregation key and OTLP export span.resource already carries the full gRPC method path for gRPC spans (e.g. /package.Service/Method). Adding grpc_method as a separate aggregation dimension or OTLP attribute is redundant and adds cardinality. Removes: - grpc_method from FixedAggregationKey - grpc_method from OtlpExactGroup - rpc.method from the OTLP data-point attributes emitted by build_attributes - GRPC_METHOD_FIELD constant and get_grpc_method() helper Co-Authored-By: Claude Sonnet 4.6 --- datadog-ipc/src/shm_stats.rs | 1 - libdd-data-pipeline/src/otlp/metrics.rs | 19 ++-------- .../src/span_concentrator/aggregation.rs | 35 +------------------ 3 files changed, 3 insertions(+), 52 deletions(-) diff --git a/datadog-ipc/src/shm_stats.rs b/datadog-ipc/src/shm_stats.rs index 82c69c9d26..a103d246a0 100644 --- a/datadog-ipc/src/shm_stats.rs +++ b/datadog-ipc/src/shm_stats.rs @@ -854,7 +854,6 @@ mod tests { http_method: "GET", http_endpoint: "/", service_source: "", - grpc_method: "", http_status_code: 200, is_synthetics_request: false, is_trace_root: true, diff --git a/libdd-data-pipeline/src/otlp/metrics.rs b/libdd-data-pipeline/src/otlp/metrics.rs index 3e05d53e19..0bf1b3eab9 100644 --- a/libdd-data-pipeline/src/otlp/metrics.rs +++ b/libdd-data-pipeline/src/otlp/metrics.rs @@ -58,7 +58,7 @@ pub fn map_stats_to_otlp_metrics( continue; }; data_points.push(json!({ - "attributes": build_attributes(group, &exact.grpc_method, is_error, resource_info, otel_trace_semantics_enabled), + "attributes": build_attributes(group, is_error, resource_info, otel_trace_semantics_enabled), "startTimeUnixNano": b.bucket.start.to_string(), "timeUnixNano": end.to_string(), "count": cell.count.to_string(), @@ -120,7 +120,6 @@ fn sketch_bucket_counts(sketch: &DDSketch) -> Vec { fn build_attributes( group: &pb::ClientGroupedStats, - grpc_method: &str, is_error: bool, resource_info: &OtlpResourceInfo, otel_trace_semantics_enabled: bool, @@ -146,7 +145,6 @@ fn build_attributes( push("span.kind", &group.span_kind); push("http.request.method", &group.http_method); push("http.route", &group.http_endpoint); - push("rpc.method", grpc_method); push("rpc.response.status_code", &group.grpc_status_code); for tag in &group.peer_tags { if let Some((k, v)) = tag.split_once(':') { @@ -346,22 +344,10 @@ mod tests { OtlpExactGroup { ok: cell(ok_ns), error: cell(err_ns), - grpc_method: String::new(), }, ) } - fn group_pair_with_grpc( - ok_ns: &[u64], - err_ns: &[u64], - grpc_method: &str, - customize: impl FnOnce(&mut pb::ClientGroupedStats), - ) -> (pb::ClientGroupedStats, OtlpExactGroup) { - let (g, mut e) = group_with_exact(ok_ns, err_ns, customize); - e.grpc_method = grpc_method.into(); - (g, e) - } - fn buckets(groups: Vec<(pb::ClientGroupedStats, OtlpExactGroup)>) -> Vec { let (stats, exact): (Vec<_>, Vec<_>) = groups.into_iter().unzip(); vec![OtlpStatsBucket { @@ -443,7 +429,7 @@ mod tests { #[test] fn data_point_attributes_and_otel_strip() { - let g_pair = group_pair_with_grpc(&[1_000_000_000], &[], "/pkg.Svc/Method", |g| { + let g_pair = group_with_exact(&[1_000_000_000], &[], |g| { g.http_status_code = 404; g.http_method = "POST".into(); g.http_endpoint = "/users/:id".into(); @@ -466,7 +452,6 @@ mod tests { assert_eq!(str_at(a, "http.request.method"), Some("POST")); assert_eq!(str_at(a, "http.route"), Some("/users/:id")); assert!(a.iter().any(|kv| kv["key"] == "http.response.status_code")); - assert_eq!(str_at(a, "rpc.method"), Some("/pkg.Svc/Method")); assert_eq!(str_at(a, "datadog.operation.name"), Some("test.op")); assert_eq!(str_at(a, "datadog.span.type"), Some("web")); assert_eq!(str_at(a, "datadog.origin"), Some("synthetics")); diff --git a/libdd-trace-stats/src/span_concentrator/aggregation.rs b/libdd-trace-stats/src/span_concentrator/aggregation.rs index c2df29ebd2..a76a90b248 100644 --- a/libdd-trace-stats/src/span_concentrator/aggregation.rs +++ b/libdd-trace-stats/src/span_concentrator/aggregation.rs @@ -24,7 +24,6 @@ const GRPC_STATUS_CODE_FIELD: &[&str] = &[ "rpc.grpc.status.code", "grpc.status.code", ]; -const GRPC_METHOD_FIELD: &[&str] = &["grpc.method.name", "rpc.method"]; /// Aggregation key fields shared across all concentrator implementations — everything /// **except** peer tags. @@ -49,7 +48,6 @@ pub struct FixedAggregationKey { pub grpc_status_code: Option, pub is_synthetics_request: bool, pub is_trace_root: bool, - pub grpc_method: T, } impl FixedAggregationKey { @@ -74,7 +72,6 @@ impl FixedAggregationKey { grpc_status_code: self.grpc_status_code, is_synthetics_request: self.is_synthetics_request, is_trace_root: self.is_trace_root, - grpc_method: f(self.grpc_method.borrow()), } } } @@ -155,17 +152,6 @@ fn get_grpc_status_code<'a>(span: &'a impl StatSpan<'a>) -> Option { None } -fn get_grpc_method<'a>(span: &'a impl StatSpan<'a>) -> &'a str { - for key in GRPC_METHOD_FIELD { - if let Some(val) = span.get_meta(key) { - if !val.is_empty() { - return val; - } - } - } - "" -} - fn grpc_status_str_to_int_value(v: &str) -> Option { if let Ok(status) = v.parse() { return Some(status); @@ -264,8 +250,6 @@ impl<'a> BorrowedAggregationKey<'a> { }; let grpc_status_code = get_grpc_status_code(span); - let grpc_method = get_grpc_method(span); - let service_source = span.get_meta(TAG_SVC_SRC).unwrap_or_default(); Self { @@ -278,7 +262,6 @@ impl<'a> BorrowedAggregationKey<'a> { http_method, http_endpoint, service_source, - grpc_method, http_status_code: status_code, grpc_status_code, is_synthetics_request: span @@ -303,7 +286,6 @@ impl From for OwnedAggregationKey { http_method: value.http_method, http_endpoint: value.http_endpoint, service_source: value.service_source, - grpc_method: String::new(), http_status_code: value.http_status_code, grpc_status_code: value.grpc_status_code.parse().ok(), is_synthetics_request: value.synthetics, @@ -392,13 +374,11 @@ pub struct OtlpExactCell { } /// Exact OK/ERROR cells for one aggregation group, in the same order as the `stats` vector -/// of the accompanying [`pb::ClientStatsBucket`]. `grpc_method` mirrors the aggregation key -/// field; it is not in the agent stats protobuf so it is surfaced here for OTLP export. +/// of the accompanying [`pb::ClientStatsBucket`]. #[derive(Debug, Clone, Default)] pub struct OtlpExactGroup { pub ok: OtlpExactCell, pub error: OtlpExactCell, - pub grpc_method: String, } /// A bucket flushed for the OTLP trace-metrics path. `exact[i]` is the exact-scalar sidecar @@ -454,7 +434,6 @@ impl StatsBucket { let mut exact = Vec::with_capacity(self.data.len()); for (k, g) in self.data { exact.push(OtlpExactGroup { - grpc_method: k.fixed.grpc_method.clone(), ok: OtlpExactCell { count: g.hits.saturating_sub(g.errors), duration_ns: g.ok_duration, @@ -830,18 +809,6 @@ mod tests { } .into_key(), ), - ( - SpanBytes { - meta: vec![("grpc.method.name".into(), "/pkg.Svc/Method".into())].into(), - ..Default::default() - }, - FixedAggregationKey { - grpc_method: "/pkg.Svc/Method".into(), - is_trace_root: true, - ..Default::default() - } - .into_key(), - ), // Span with grpc status from meta as numeric string ( SpanBytes {