feat(secrets): support JSON-structured Secrets Manager secrets via DD_API_KEY_SECRET_JSON_KEY

Add DD_API_KEY_SECRET_JSON_KEY env var that, when set alongside DD_API_KEY_SECRET_ARN,
parses the secret value as JSON and extracts the specified field as the API key.
This allows users with key/value JSON secrets to avoid duplicating or exposing their API key.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: litianningdatadog

bottlecap/src/config/env.rs

@@ -381,6 +381,12 @@ pub struct EnvConfig {
     /// The AWS ARN of the secret containing the Datadog API key.
     #[serde(deserialize_with = "deserialize_optional_string")]
     pub api_key_secret_arn: Option<String>,
+    /// @env `DD_API_KEY_SECRET_JSON_KEY`
+    ///
+    /// When set, the secret fetched via `DD_API_KEY_SECRET_ARN` is parsed as JSON
+    /// and the value of this key is used as the API key.
+    #[serde(deserialize_with = "deserialize_optional_string")]
+    pub api_key_secret_json_key: Option<String>,
     /// @env `DD_KMS_API_KEY`
     ///
     /// The AWS KMS API key to use for the Datadog Agent.
@@ -661,6 +667,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
 
     // AWS Lambda
     merge_string!(config, env_config, api_key_secret_arn);
+    merge_string!(config, env_config, api_key_secret_json_key);
     merge_string!(config, env_config, kms_api_key);
     merge_string!(config, env_config, api_key_ssm_arn);
     merge_option_to_value!(config, env_config, serverless_logs_enabled);
@@ -871,6 +878,7 @@ mod tests {
                 "DD_API_KEY_SECRET_ARN",
                 "arn:aws:secretsmanager:region:account:secret:datadog-api-key",
             );
+            jail.set_env("DD_API_KEY_SECRET_JSON_KEY", "apiKey");
             jail.set_env("DD_KMS_API_KEY", "test-kms-key");
             jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false");
             jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,60000");
@@ -1026,6 +1034,7 @@ mod tests {
                 dogstatsd_queue_size: Some(2048),
                 api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key"
                     .to_string(),
+                api_key_secret_json_key: "apiKey".to_string(),
                 kms_api_key: "test-kms-key".to_string(),
                 api_key_ssm_arn: String::default(),
                 serverless_logs_enabled: false,

bottlecap/src/config/mod.rs

@@ -354,6 +354,7 @@ pub struct Config {
     pub api_key_secret_arn: String,
     pub kms_api_key: String,
     pub api_key_ssm_arn: String,
+    pub api_key_secret_json_key: String,
     pub serverless_logs_enabled: bool,
     pub serverless_flush_strategy: FlushStrategy,
     pub enhanced_metrics: bool,
@@ -469,6 +470,7 @@ impl Default for Config {
             api_key_secret_arn: String::default(),
             kms_api_key: String::default(),
             api_key_ssm_arn: String::default(),
+            api_key_secret_json_key: String::default(),
             serverless_logs_enabled: true,
             serverless_flush_strategy: FlushStrategy::Default,
             enhanced_metrics: true,

bottlecap/src/config/yaml.rs

@@ -112,6 +112,8 @@ pub struct YamlConfig {
     #[serde(deserialize_with = "deserialize_optional_string")]
     pub api_key_secret_arn: Option<String>,
     #[serde(deserialize_with = "deserialize_optional_string")]
+    pub api_key_secret_json_key: Option<String>,
+    #[serde(deserialize_with = "deserialize_optional_string")]
     pub kms_api_key: Option<String>,
     #[serde(deserialize_with = "deserialize_optional_bool_from_anything")]
     pub serverless_logs_enabled: Option<bool>,
@@ -693,6 +695,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) {
 
     // AWS Lambda
     merge_string!(config, yaml_config, api_key_secret_arn);
+    merge_string!(config, yaml_config, api_key_secret_json_key);
     merge_string!(config, yaml_config, kms_api_key);
 
     // Handle serverless_logs_enabled with OR logic: if either logs_enabled or serverless_logs_enabled is true, enable logs
@@ -875,6 +878,7 @@ otlp_config:
 
 # AWS Lambda
 api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key"
+api_key_secret_json_key: "apiKey"
 kms_api_key: "test-kms-key"
 serverless_logs_enabled: false
 serverless_flush_strategy: "periodically,60000"
@@ -1008,6 +1012,7 @@ api_security_sample_delay: 60 # Seconds
                 otlp_config_logs_enabled: true,
                 api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key"
                     .to_string(),
+                api_key_secret_json_key: "apiKey".to_string(),
                 kms_api_key: "test-kms-key".to_string(),
                 api_key_ssm_arn: String::default(),
                 serverless_logs_enabled: false,

bottlecap/src/secrets/decrypt.rs

@@ -83,6 +83,7 @@ pub async fn resolve_secrets(config: Arc<Config>, aws_config: Arc<AwsConfig>) ->
             decrypt_aws_sm(
                 &client,
                 config.api_key_secret_arn.clone(),
+                &config.api_key_secret_json_key,
                 aws_config,
                 &aws_credentials,
             )
@@ -194,6 +195,7 @@ async fn decrypt_aws_kms(
 async fn decrypt_aws_sm(
     client: &Client,
     secret_arn: String,
+    json_key: &str,
     aws_config: Arc<AwsConfig>,
     aws_credentials: &AwsCredentials,
 ) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
@@ -217,9 +219,25 @@ async fn decrypt_aws_sm(
     );
 
     let v = request(json_body, headers?, client).await?;
+    extract_secret_string(&v, json_key)
+}
 
+fn extract_secret_string(
+    v: &Value,
+    json_key: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
     if let Some(secret_string) = v["SecretString"].as_str() {
-        Ok(secret_string.to_string())
+        if json_key.is_empty() {
+            return Ok(secret_string.to_string());
+        }
+        let parsed: Value = serde_json::from_str(secret_string)?;
+        let extracted = parsed[json_key].as_str().ok_or_else(|| {
+            Error::new(
+                std::io::ErrorKind::InvalidData,
+                format!("JSON key '{json_key}' not found in secret or is not a string"),
+            )
+        })?;
+        Ok(extracted.to_string())
     } else {
         Err(Error::new(std::io::ErrorKind::InvalidData, v.to_string()).into())
     }
@@ -401,6 +419,31 @@ mod tests {
     use super::*;
     use chrono::{NaiveDateTime, TimeZone};
 
+    fn make_sm_response(secret_string: &str) -> Value {
+        serde_json::json!({ "SecretString": secret_string })
+    }
+
+    #[test]
+    fn test_json_secret_extraction() {
+        let v = make_sm_response(r#"{"apiKey":"abc123"}"#);
+        let result = extract_secret_string(&v, "apiKey").expect("should extract apiKey");
+        assert_eq!(result, "abc123");
+    }
+
+    #[test]
+    fn test_json_secret_missing_key() {
+        let v = make_sm_response(r#"{"apiKey":"abc123"}"#);
+        let err = extract_secret_string(&v, "wrongKey").expect_err("should fail on missing key");
+        assert!(err.to_string().contains("wrongKey"));
+    }
+
+    #[test]
+    fn test_plain_secret_unaffected() {
+        let v = make_sm_response("abc123");
+        let result = extract_secret_string(&v, "").expect("should return raw value");
+        assert_eq!(result, "abc123");
+    }
+
     #[test]
     fn key_cleanup() {
         let key = clean_api_key(Some(" 32alxcxf\n".to_string()));