refactor(span)!: use VecMap for meta, metrics and meta_struct for v04 spans (#2043)

Depends on #2022 and #2049.

# What does this PR do?

This PR continues the native span performance work required to land native spans in dd-trace-js. Follow-up of #2022. Actually use `VecMap` in the `Span` data structure.

# Motivation

Performance improvement following dd-trace-js native span experiments. See #2022.

# Additional considerations

There are some deduplication design choices. We deduplicate before serializing to the agent, because while the current agent implementation would support duplicate keys in a map (with the same semantics of "last one wins"), this is not part of the spec/API, so we can't rely on it. We also deduplicate defensively in the msgpack encoder, but it should already be deduped at this stage.

For byte estimate (`byte_size()`), we make the choice of not deduplicating. This means potentially overestimating the size of a chunk (and reaching the limit sooner), in exchange of delaying the deduplication to serialization time. 

# How to test the change?

Run tests.

Co-authored-by: yann.hamdaoui <yann.hamdaoui@datadoghq.com>

Author: yannham

datadog-sidecar-ffi/Cargo.toml

@@ -35,6 +35,7 @@ libdd-crashtracker-ffi = { path = "../libdd-crashtracker-ffi", features = ["coll
 
 [dev-dependencies]
 http = "1.1"
+libdd-trace-utils = { path = "../libdd-trace-utils", features = ["test-utils"] }
 tempfile = { version = "3.3" }
 
 [lints.rust]

datadog-sidecar-ffi/src/span.rs

@@ -5,6 +5,7 @@ use libdd_common_ffi::slice::{AsBytes, CharSlice};
 use libdd_tinybytes::{Bytes, BytesString};
 use libdd_trace_utils::span::v04::{
     AttributeAnyValueBytes, AttributeArrayValueBytes, SpanBytes, SpanEventBytes, SpanLinkBytes,
+    VecMap,
 };
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -51,25 +52,35 @@ fn insert_hashmap<V>(map: &mut HashMap<BytesString, V>, key: CharSlice, value: V
 }
 
 #[inline]
-fn remove_hashmap<V>(map: &mut HashMap<BytesString, V>, key: CharSlice) {
+fn insert_vec_map<V>(map: &mut VecMap<BytesString, V>, key: CharSlice, value: V) {
+    if key.is_empty() {
+        return;
+    }
+    let bytes_str_key = convert_char_slice_to_bytes_string(key);
+    map.insert(bytes_str_key, value);
+}
+
+#[inline]
+fn remove_vec_map_slow<V>(map: &mut VecMap<BytesString, V>, key: CharSlice) {
     let bytes_str_key = convert_char_slice_to_bytes_string(key);
-    map.remove(&bytes_str_key);
+    map.remove_slow(&bytes_str_key);
 }
 
 #[inline]
-fn exists_hashmap<V>(map: &HashMap<BytesString, V>, key: CharSlice) -> bool {
+fn exists_vec_map<V>(map: &VecMap<BytesString, V>, key: CharSlice) -> bool {
     let bytes_str_key = convert_char_slice_to_bytes_string(key);
     map.contains_key(&bytes_str_key)
 }
 
 /// The return value is an owned array of slices (`Box<[CharSlice<'a>]>`) that must be dropped
 /// explicitly.
-fn get_hashmap_keys<'a, V>(
-    map: &'a HashMap<BytesString, V>,
+fn get_vec_map_keys<'a, V>(
+    map: &'a VecMap<BytesString, V>,
     out_count: &mut usize,
 ) -> *mut CharSlice<'a> {
-    let mut keys: Vec<&str> = map.keys().map(|b| b.as_str()).collect();
+    let mut keys: Vec<&str> = map.iter().map(|(k, _)| k.as_str()).collect();
     keys.sort_unstable();
+    keys.dedup();
 
     let slices: Box<[CharSlice]> = keys
         .iter()
@@ -182,8 +193,8 @@ pub extern "C" fn ddog_trace_new_span_with_capacities(
     new_vector_push(
         trace,
         SpanBytes {
-            meta: HashMap::with_capacity(meta_size),
-            metrics: HashMap::with_capacity(metrics_size),
+            meta: VecMap::with_capacity(meta_size),
+            metrics: VecMap::with_capacity(metrics_size),
             ..SpanBytes::default()
         },
     )
@@ -324,7 +335,7 @@ pub extern "C" fn ddog_get_span_error(span: &mut SpanBytes) -> i32 {
 
 #[no_mangle]
 pub extern "C" fn ddog_add_span_meta(span: &mut SpanBytes, key: CharSlice, value: CharSlice) {
-    insert_hashmap(
+    insert_vec_map(
         &mut span.meta,
         key,
         BytesString::from_slice(value.as_bytes()).unwrap_or_default(),
@@ -333,7 +344,7 @@ pub extern "C" fn ddog_add_span_meta(span: &mut SpanBytes, key: CharSlice, value
 
 #[no_mangle]
 pub extern "C" fn ddog_del_span_meta(span: &mut SpanBytes, key: CharSlice) {
-    remove_hashmap(&mut span.meta, key);
+    remove_vec_map_slow(&mut span.meta, key);
 }
 
 #[no_mangle]
@@ -355,7 +366,7 @@ pub extern "C" fn ddog_get_span_meta<'a>(
 
 #[no_mangle]
 pub extern "C" fn ddog_has_span_meta(span: &mut SpanBytes, key: CharSlice) -> bool {
-    exists_hashmap(&span.meta, key)
+    exists_vec_map(&span.meta, key)
 }
 
 /// The return value is an owned array of slices (`Box<[CharSlice]>`) that must be freed explicitly
@@ -365,17 +376,17 @@ pub extern "C" fn ddog_span_meta_get_keys<'a>(
     span: &'a mut SpanBytes,
     out_count: &mut usize,
 ) -> *mut CharSlice<'a> {
-    get_hashmap_keys(&span.meta, out_count)
+    get_vec_map_keys(&span.meta, out_count)
 }
 
 #[no_mangle]
 pub extern "C" fn ddog_add_span_metrics(span: &mut SpanBytes, key: CharSlice, val: f64) {
-    insert_hashmap(&mut span.metrics, key, val);
+    insert_vec_map(&mut span.metrics, key, val);
 }
 
 #[no_mangle]
 pub extern "C" fn ddog_del_span_metrics(span: &mut SpanBytes, key: CharSlice) {
-    remove_hashmap(&mut span.metrics, key);
+    remove_vec_map_slow(&mut span.metrics, key);
 }
 
 #[no_mangle]
@@ -396,20 +407,20 @@ pub extern "C" fn ddog_get_span_metrics(
 
 #[no_mangle]
 pub extern "C" fn ddog_has_span_metrics(span: &mut SpanBytes, key: CharSlice) -> bool {
-    exists_hashmap(&span.metrics, key)
+    exists_vec_map(&span.metrics, key)
 }
 
 #[no_mangle]
 pub extern "C" fn ddog_span_metrics_get_keys<'a>(
     span: &'a mut SpanBytes,
     out_count: &mut usize,
 ) -> *mut CharSlice<'a> {
-    get_hashmap_keys(&span.metrics, out_count)
+    get_vec_map_keys(&span.metrics, out_count)
 }
 
 #[no_mangle]
 pub extern "C" fn ddog_add_span_meta_struct(span: &mut SpanBytes, key: CharSlice, val: CharSlice) {
-    insert_hashmap(
+    insert_vec_map(
         &mut span.meta_struct,
         key,
         Bytes::copy_from_slice(val.as_bytes()),
@@ -418,7 +429,7 @@ pub extern "C" fn ddog_add_span_meta_struct(span: &mut SpanBytes, key: CharSlice
 
 #[no_mangle]
 pub extern "C" fn ddog_del_span_meta_struct(span: &mut SpanBytes, key: CharSlice) {
-    remove_hashmap(&mut span.meta_struct, key);
+    remove_vec_map_slow(&mut span.meta_struct, key);
 }
 
 #[no_mangle]
@@ -438,7 +449,7 @@ pub extern "C" fn ddog_get_span_meta_struct<'a>(
 
 #[no_mangle]
 pub extern "C" fn ddog_has_span_meta_struct(span: &mut SpanBytes, key: CharSlice) -> bool {
-    exists_hashmap(&span.meta_struct, key)
+    exists_vec_map(&span.meta_struct, key)
 }
 
 /// The return value is an array of slices (`Box<[CharSlice]>`) that must be freed explicitly
@@ -448,7 +459,7 @@ pub extern "C" fn ddog_span_meta_struct_get_keys<'a>(
     span: &'a mut SpanBytes,
     out_count: &mut usize,
 ) -> *mut CharSlice<'a> {
-    get_hashmap_keys(&span.meta_struct, out_count)
+    get_vec_map_keys(&span.meta_struct, out_count)
 }
 
 /// # Safety
@@ -461,7 +472,7 @@ pub unsafe extern "C" fn ddog_span_free_keys_ptr(keys_ptr: *mut CharSlice<'_>, c
         return;
     }
 
-    // Safety: all `xxx_get_keys()` functions return from `get_hashmap_keys()`, which returns a
+    // Safety: all `xxx_get_keys()` functions return from `get_vec_map_keys()`, which returns a
     // `Box<[T]>`. It is an official guarantee of `Vec` that this can be freely converted to and
     // from `Box<[T]>` when `len == capacity`.
     unsafe {

datadog-sidecar-ffi/tests/span.rs

@@ -186,7 +186,7 @@ fn test_span_debug_log_output() {
     ddog_set_span_name(span, CharSlice::from("debug-span"));
     let debug_output = ddog_span_debug_log(span);
 
-    let expected_output = CharSlice::from("Span { service: , name: debug-span, resource: , type: , trace_id: 0, span_id: 0, parent_id: 0, start: 0, duration: 0, error: 0, meta: {}, metrics: {}, meta_struct: {}, span_links: [], span_events: [] }");
+    let expected_output = CharSlice::from("Span { service: , name: debug-span, resource: , type: , trace_id: 0, span_id: 0, parent_id: 0, start: 0, duration: 0, error: 0, meta: VecMap { data: [], deduped: false }, metrics: VecMap { data: [], deduped: false }, meta_struct: VecMap { data: [], deduped: false }, span_links: [], span_events: [] }");
 
     assert_eq!(debug_output, expected_output);
 
@@ -343,12 +343,13 @@ fn test_full_span() {
         start: 4,
         duration: 5,
         error: 6,
-        meta: HashMap::from([(get_bytes_str("meta_key"), get_bytes_str("meta_value"))]),
-        metrics: HashMap::from([(get_bytes_str("metric_key"), 1.0)]),
-        meta_struct: HashMap::from([(
+        meta: vec![(get_bytes_str("meta_key"), get_bytes_str("meta_value"))].into(),
+        metrics: vec![(get_bytes_str("metric_key"), 1.0)].into(),
+        meta_struct: vec![(
             get_bytes_str("meta_struct_key"),
             get_bytes("meta_struct_value"),
-        )]),
+        )]
+        .into(),
         span_links: vec![SpanLinkBytes {
             trace_id: 10,
             span_id: 20,

libdd-data-pipeline-ffi/src/tracer.rs

@@ -499,7 +499,8 @@ mod tests {
             ddog_tracer_span_set_meta(Some(&mut *span), cs("k"), cs("v1"));
             ddog_tracer_span_set_meta(Some(&mut *span), cs("k"), cs("v2"));
 
-            assert_eq!(span.0.meta.len(), 1);
+            // After the introduction of `VecMap`, the length is still 2, as the data structure
+            // tolerates duplicate entries.
             assert_eq!(span.0.meta.get("k").unwrap().as_ref(), "v2");
 
             ddog_tracer_span_free(span);

libdd-data-pipeline/benches/trace_buffer.rs

@@ -1,7 +1,6 @@
 // Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
-use std::collections::HashMap;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
@@ -14,6 +13,7 @@ use libdd_data_pipeline::trace_exporter::{
 use libdd_shared_runtime::SharedRuntime;
 use libdd_tinybytes::BytesString;
 use libdd_trace_utils::span::v04::SpanBytes;
+use libdd_trace_utils::span::vec_map::VecMap;
 
 // Number of chunks each sender thread sends per benchmark iteration.
 const CHUNKS_PER_SENDER: usize = 900;
@@ -34,18 +34,20 @@ fn make_span() -> SpanBytes {
         start: 1_700_000_000_000_000_000_i64,
         duration: 5_000_000_i64,
         error: 0,
-        meta: HashMap::from_iter([
+        meta: vec![
             (bs("env"), bs("prod")),
             (bs("version"), bs("1.0.0")),
             (bs("http.method"), bs("GET")),
             (bs("http.url"), bs("/api/v1/users")),
             (bs("peer.service"), bs("users-service")),
-        ]),
-        metrics: HashMap::from_iter([
+        ]
+        .into(),
+        metrics: vec![
             (bs("_sampling_priority_v1"), 1.0_f64),
             (bs("_dd.agent_psr"), 1.0_f64),
-        ]),
-        meta_struct: HashMap::new(),
+        ]
+        .into(),
+        meta_struct: VecMap::new(),
         span_links: vec![],
         span_events: vec![],
     }

libdd-data-pipeline/src/trace_buffer/mod.rs

@@ -42,13 +42,16 @@ where
         size += self.resource.as_ref().len();
         size += self.r#type.as_ref().len();
 
-        for (k, v) in &self.meta {
+        // We expect VecMaps to be already deduped at this point, so `defensive_dedup` should be
+        // cheap (and alloc-free). In the future we could relax the check and accept non-deduped
+        // VecMap, trading over-estimating the size of a span for less work.
+        for (k, v) in self.meta.defensive_dedup().iter() {
             size += k.as_ref().len() + v.as_ref().len();
         }
-        for k in self.metrics.keys() {
+        for (k, _) in self.metrics.defensive_dedup().iter() {
             size += k.as_ref().len() + 8;
         }
-        for (k, v) in &self.meta_struct {
+        for (k, v) in self.meta_struct.defensive_dedup().iter() {
             size += k.as_ref().len() + v.as_ref().len();
         }
         for link in &self.span_links {

libdd-data-pipeline/src/trace_exporter/mod.rs

@@ -612,6 +612,12 @@ impl<C: HttpClientCapability + SleepCapability + MaybeSend + Sync + 'static> Tra
             self.client_computed_top_level,
         );
 
+        for chunk in &mut traces {
+            for span in chunk.iter_mut() {
+                span.dedup();
+            }
+        }
+
         // OTLP path: send sampled traces via OTLP when an OTLP endpoint is configured.
         // Unlike the agent path, there is no downstream agent to drop unsampled traces,
         // so drop_chunks is always called here regardless of whether stats are enabled.

libdd-trace-stats/benches/span_concentrator_bench.rs

@@ -1,25 +1,22 @@
 // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
-use std::{
-    collections::HashMap,
-    time::{self, Duration, SystemTime},
-};
+use std::time::{self, Duration, SystemTime};
 
 use criterion::{criterion_group, Criterion};
 use libdd_trace_stats::span_concentrator::SpanConcentrator;
-use libdd_trace_utils::span::v04::SpanBytes;
+use libdd_trace_utils::span::v04::{SpanBytes, VecMap};
 
 fn get_bucket_start(now: SystemTime, n: u64) -> i64 {
     let start = now.duration_since(time::UNIX_EPOCH).unwrap() + Duration::from_secs(10 * n);
     start.as_nanos() as i64
 }
 
 fn get_span(now: SystemTime, trace_id: u64, span_id: u64) -> SpanBytes {
-    let mut metrics = HashMap::from([("_dd.measured".into(), 1.0)]);
+    let mut metrics: VecMap<_, _> = vec![("_dd.measured".into(), 1.0)].into();
     if span_id == 1 {
         metrics.insert("_dd.top_level".into(), 1.0);
     }
-    let mut meta = HashMap::from([("db_name".into(), "postgres".into())]);
+    let mut meta: VecMap<_, _> = vec![("db_name".into(), "postgres".into())].into();
     if span_id.is_multiple_of(3) {
         meta.insert("bucket_s3".into(), "aws_bucket".into());
     }