Skip to content

Commit 155a5a3

Browse files
authored
[SVLS-7934] feat: Support TLS certificate for trace/stats flusher (#961)
## Problem A customer reported that their Lambda is behind a proxy, and the Rust-based extension can't send traces to Datadog via the proxy, while the previous go-based extension worked. ## This PR Supports the env var `DD_TLS_CERT_FILE`: The path to a file of concatenated CA certificates in PEM format. Example: `DD_TLS_CERT_FILE=/opt/ca-cert.pem`, so the when the extension flushes traces/stats to Datadog, the HTTP client created can load and use this cert, and connect the proxy properly. ## Testing ### Steps 1. Create a Lambda in a VPC with an NGINX proxy. 2. Add a layer to the Lambda, which includes the CA certificate `ca-cert.pem` 3. Set env vars: - `DD_TLS_CERT_FILE=/opt/ca-cert.pem` - `DD_PROXY_HTTPS=http://10.0.0.30:3128`, where `10.0.0.30` is the private IP of the proxy EC2 instance - `DD_LOG_LEVEL=debug` 4. Update routing rules of security groups so the Lambda can reach `http://10.0.0.30:3128` 5. Invoke the Lambda ### Result **Before** Trace flush failed with error logs: > DD_EXTENSION | ERROR | Max retries exceeded, returning request error error=Network error: client error (Connect) attempts=1 DD_EXTENSION | ERROR | TRACES | Request failed: No requests sent **After** Trace flush is successful: > DD_EXTENSION | DEBUG | TRACES | Flushing 1 traces DD_EXTENSION | DEBUG | TRACES | Added root certificate from /opt/ca-cert.pem DD_EXTENSION | DEBUG | TRACES | Proxy connector created with proxy: Some("http://10.0.0.30:3128") DD_EXTENSION | DEBUG | Sending with retry url=https://trace.agent.datadoghq.com/api/v0.2/traces payload_size=1120 max_retries=1 DD_EXTENSION | DEBUG | Received response status=202 Accepted attempt=1 DD_EXTENSION | DEBUG | Request succeeded status=202 Accepted attempts=1 DD_EXTENSION | DEBUG | TRACES | Flushing took 1609 ms ## Notes This fix only covers trace flusher and stats flusher, which use `ServerlessTraceFlusher::get_http_client()` to create the HTTP client. It doesn't cover logs flusher and proxy flusher, which use a different function (http.rs:get_client()) to create the HTTP client. However, logs flushing was successful in my tests, even if no certificate was added. We can come back to logs/proxy flusher if someone reports an error.
1 parent 90acf2a commit 155a5a3

3 files changed

Lines changed: 15 additions & 0 deletions

File tree

env.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ pub struct EnvConfig {
7575
/// The transport type to use for sending logs. Possible values are "auto" or "http1".
7676
#[serde(deserialize_with = "deserialize_optional_string")]
7777
pub http_protocol: Option<String>,
78+
/// @env `DD_TLS_CERT_FILE`
79+
/// The path to a file of concatenated CA certificates in PEM format.
80+
/// Example: `/opt/ca-cert.pem`
81+
#[serde(deserialize_with = "deserialize_optional_string")]
82+
pub tls_cert_file: Option<String>,
7883

7984
// Metrics
8085
/// @env `DD_DD_URL`
@@ -466,6 +471,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
466471
merge_option!(config, env_config, proxy_https);
467472
merge_vec!(config, env_config, proxy_no_proxy);
468473
merge_option!(config, env_config, http_protocol);
474+
merge_option!(config, env_config, tls_cert_file);
469475

470476
// Endpoints
471477
merge_string!(config, env_config, dd_url);
@@ -695,6 +701,7 @@ mod tests {
695701
jail.set_env("DD_PROXY_HTTPS", "https://proxy.example.com");
696702
jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1");
697703
jail.set_env("DD_HTTP_PROTOCOL", "http1");
704+
jail.set_env("DD_TLS_CERT_FILE", "/opt/ca-cert.pem");
698705

699706
// Metrics
700707
jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com");
@@ -850,6 +857,7 @@ mod tests {
850857
proxy_https: Some("https://proxy.example.com".to_string()),
851858
proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()],
852859
http_protocol: Some("http1".to_string()),
860+
tls_cert_file: Some("/opt/ca-cert.pem".to_string()),
853861
dd_url: "https://metrics.datadoghq.com".to_string(),
854862
url: "https://app.datadoghq.com".to_string(),
855863
additional_endpoints: HashMap::from([

mod.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ pub struct Config {
252252
pub proxy_https: Option<String>,
253253
pub proxy_no_proxy: Vec<String>,
254254
pub http_protocol: Option<String>,
255+
pub tls_cert_file: Option<String>,
255256

256257
// Endpoints
257258
pub dd_url: String,
@@ -366,6 +367,7 @@ impl Default for Config {
366367
proxy_https: None,
367368
proxy_no_proxy: vec![],
368369
http_protocol: None,
370+
tls_cert_file: None,
369371

370372
// Endpoints
371373
dd_url: String::default(),

yaml.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ pub struct YamlConfig {
5353
pub dd_url: Option<String>,
5454
#[serde(deserialize_with = "deserialize_optional_string")]
5555
pub http_protocol: Option<String>,
56+
#[serde(deserialize_with = "deserialize_optional_string")]
57+
pub tls_cert_file: Option<String>,
5658

5759
// Endpoints
5860
#[serde(deserialize_with = "deserialize_additional_endpoints")]
@@ -417,6 +419,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
417419
merge_option!(config, proxy_https, yaml_config.proxy, https);
418420
merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy);
419421
merge_option!(config, yaml_config, http_protocol);
422+
merge_option!(config, yaml_config, tls_cert_file);
420423

421424
// Endpoints
422425
merge_hashmap!(config, yaml_config, additional_endpoints);
@@ -747,6 +750,7 @@ proxy:
747750
no_proxy: ["localhost", "127.0.0.1"]
748751
dd_url: "https://metrics.datadoghq.com"
749752
http_protocol: "http1"
753+
tls_cert_file: "/opt/ca-cert.pem"
750754
751755
# Endpoints
752756
additional_endpoints:
@@ -882,6 +886,7 @@ api_security_sample_delay: 60 # Seconds
882886
proxy_https: Some("https://proxy.example.com".to_string()),
883887
proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()],
884888
http_protocol: Some("http1".to_string()),
889+
tls_cert_file: Some("/opt/ca-cert.pem".to_string()),
885890
dd_url: "https://metrics.datadoghq.com".to_string(),
886891
url: String::new(), // doesnt exist in yaml
887892
additional_endpoints: HashMap::from([

0 commit comments

Comments
 (0)