feat: dual shipping APM support (#735)

Adds support for dual shipping traces to endpoints configured using the
`apm_config` YAML or `DD_APM_CONFIG_ADDITIONAL_ENDPOINTS` env var
config.

#### Additional Notes:
- Bumped libdatadog (and serverless-components) to include
DataDog/libdatadog#1139
- Adds configuration option to set compression level for trace payloads

Author: shreyamalpani

bottlecap/Cargo.lock

@@ -57,9 +57,9 @@ datadog-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev =
 datadog-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" , features = ["mini_agent"] }
 datadog-trace-normalization = { git = "https://github.com/DataDog/libdatadog",  rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2" }
 datadog-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "8a49c7df2d9cbf05118bfd5b85772676f71b34f2"  }
-dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "985120329d0ba96c1ec8d719cc38e1f7ce11a092", default-features = false }
-datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78" }
-datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "c3d8ed4f90591c6958921145d485463860307f78", default-features = false }
+dogstatsd = { git = "https://github.com/DataDog/serverless-components", rev = "d131de8419c191ce21c91bb30b5915c4d8a2cc5a", default-features = false }
+datadog-trace-agent = { git = "https://github.com/DataDog/serverless-components", rev = "d131de8419c191ce21c91bb30b5915c4d8a2cc5a" }
+datadog-fips = { git = "https://github.com/DataDog/serverless-components", rev = "d131de8419c191ce21c91bb30b5915c4d8a2cc5a", default-features = false }
 axum = { version = "0.8.4", default-features = false, features = ["default"] }
 
 [dev-dependencies]

bottlecap/Cargo.toml

@@ -108,7 +108,6 @@ log,https://github.com/rust-lang/log,MIT OR Apache-2.0,The Rust Project Develope
 matchers,https://github.com/hawkw/matchers,MIT,Eliza Weisman <eliza@buoyant.io>
 matchit,https://github.com/ibraheemdev/matchit,MIT AND BSD-3-Clause,Ibraheem Ahmed <ibraheem@ibraheem.ca>
 memchr,https://github.com/BurntSushi/memchr,Unlicense OR MIT,"Andrew Gallant <jamslam@gmail.com>, bluss"
-memfd,https://github.com/lucab/memfd-rs,MIT OR Apache-2.0,"Luca Bruno <lucab@lucabruno.net>, Simonas Kazlauskas <memfd@kazlauskas.me>"
 mime,https://github.com/hyperium/mime,MIT OR Apache-2.0,Sean McArthur <sean@seanmonstar.com>
 miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi <daniil.liferenko@gmail.com>, oyvindln <oyvindln@users.noreply.github.com>, Rich Geldreich richgel99@gmail.com"
 mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche <me@carllerche.com>, Thomas de Zeeuw <thomasdezeeuw@gmail.com>, Tokio Contributors <team@tokio.rs>"

bottlecap/LICENSE-3rdparty.csv

@@ -1057,11 +1057,11 @@ fn start_trace_agent(
     let stats_processor = Arc::new(stats_processor::ServerlessStatsProcessor {});
 
     // Traces
-    let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher {
-        aggregator: trace_aggregator.clone(),
-        config: Arc::clone(config),
-        api_key_factory: Arc::clone(api_key_factory),
-    });
+    let trace_flusher = Arc::new(trace_flusher::ServerlessTraceFlusher::new(
+        trace_aggregator.clone(),
+        config.clone(),
+        api_key_factory.clone(),
+    ));
 
     let obfuscation_config = obfuscation_config::ObfuscationConfig {
         tag_replace_rules: config.apm_replace_tags.clone(),

bottlecap/src/bin/bottlecap/main.rs

@@ -163,9 +163,21 @@ pub struct EnvConfig {
     /// @env `DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS`
     #[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
     pub apm_config_obfuscation_http_remove_paths_with_digits: Option<bool>,
+    /// @env `DD_APM_CONFIG_COMPRESSION_LEVEL`
+    ///
+    /// The Agent compresses traces before sending them. The `compression_level` parameter
+    /// accepts values from 0 (no compression) to 9 (maximum compression but
+    /// higher resource usage).
+    pub apm_config_compression_level: Option<i32>,
     /// @env `DD_APM_FEATURES`
     #[serde(deserialize_with = "deserialize_array_from_comma_separated_string")]
     pub apm_features: Vec<String>,
+    /// @env `DD_APM_ADDITIONAL_ENDPOINTS`
+    ///
+    /// Additional endpoints to send traces to.
+    /// <https://docs.datadoghq.com/agent/configuration/dual-shipping/?tab=helm#environment-variable-configuration-1>
+    #[serde(deserialize_with = "deserialize_additional_endpoints")]
+    pub apm_additional_endpoints: HashMap<String, Vec<String>>,
     //
     // Trace Propagation
     /// @env `DD_TRACE_PROPAGATION_STYLE`
@@ -347,7 +359,9 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
         env_config,
         apm_config_obfuscation_http_remove_paths_with_digits
     );
+    merge_option_to_value!(config, env_config, apm_config_compression_level);
     merge_vec!(config, env_config, apm_features);
+    merge_hashmap!(config, env_config, apm_additional_endpoints);
 
     // Trace Propagation
     merge_vec!(config, env_config, trace_propagation_style);
@@ -533,10 +547,12 @@ mod tests {
                 "DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS",
                 "true",
             );
+            jail.set_env("DD_APM_CONFIG_COMPRESSION_LEVEL", "3");
             jail.set_env(
                 "DD_APM_FEATURES",
                 "enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind",
             );
+            jail.set_env("DD_APM_ADDITIONAL_ENDPOINTS", "{\"https://trace.agent.datadoghq.com\": [\"apikey2\", \"apikey3\"], \"https://trace.agent.datadoghq.eu\": [\"apikey4\"]}");
 
             // Trace Propagation
             jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog");
@@ -673,10 +689,21 @@ mod tests {
                 ),
                 apm_config_obfuscation_http_remove_query_string: true,
                 apm_config_obfuscation_http_remove_paths_with_digits: true,
+                apm_config_compression_level: 3,
                 apm_features: vec![
                     "enable_otlp_compute_top_level_by_span_kind".to_string(),
                     "enable_stats_by_span_kind".to_string(),
                 ],
+                apm_additional_endpoints: HashMap::from([
+                    (
+                        "https://trace.agent.datadoghq.com".to_string(),
+                        vec!["apikey2".to_string(), "apikey3".to_string()],
+                    ),
+                    (
+                        "https://trace.agent.datadoghq.eu".to_string(),
+                        vec!["apikey4".to_string()],
+                    ),
+                ]),
                 trace_propagation_style: vec![TracePropagationStyle::Datadog],
                 trace_propagation_style_extract: vec![TracePropagationStyle::B3],
                 trace_propagation_extract_first: true,