Merge branch 'main' into zarir/base_service

Author: zarirhamza

.github/workflows/nightly-serverless-init.yml

@@ -36,9 +36,11 @@ jobs:
         run: |
           STAMP=$(date -u +%Y%m%d)
           SHORT_SHA=$(git -C datadog-agent rev-parse --short=8 HEAD)
+          AGENT_VERSION=$(grep -m 1 -E '^[0-9]+\.[0-9]+\.[0-9]+$' datadog-agent/CHANGELOG.rst) || { echo "ERROR: could not detect agent version from datadog-agent's CHANGELOG.rst"; exit 1; }
           echo "stamp=${STAMP}" >> "$GITHUB_OUTPUT"
           echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
           echo "version=nightly-${STAMP}-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          echo "agent_version=${AGENT_VERSION}" >> "$GITHUB_OUTPUT"
 
       # Pin QEMU to a known-good version. See release-serverless-init.yml
       # and test-qemu-versions.yml for context on QEMU breakage history.
@@ -56,6 +58,7 @@ jobs:
         env:
           AGENT_PATH: datadog-agent
           VERSION: ${{ steps.meta.outputs.version }}
+          AGENT_VERSION: ${{ steps.meta.outputs.agent_version }}
           SERVERLESS_INIT: "true"
           ALPINE: ${{ matrix.arrays.isAlpine }}
 

.github/workflows/release-serverless-init.yml

@@ -22,7 +22,7 @@ on:
           - "no"
       agentVersion:
         type: string
-        description: Datadog agent version
+        description: Datadog agent version (default latest release tag from Datadog agent branch)
       agentBranch:
         type: string
         description: Datadog agent branch or tag name (default main)
@@ -53,6 +53,15 @@ jobs:
           ref: ${{ github.event.inputs.agentBranch }}
           path: datadog-agent
 
+      - name: Compute agent version
+        id: meta
+        run: |
+          AGENT_VERSION="${{ github.event.inputs.agentVersion }}"
+          if [ -z "$AGENT_VERSION" ]; then
+            AGENT_VERSION=$(grep -m 1 -E '^[0-9]+\.[0-9]+\.[0-9]+$' datadog-agent/CHANGELOG.rst) || { echo "ERROR: could not detect agent version from datadog-agent's CHANGELOG.rst; set the Datadog agent version manually"; exit 1; }
+          fi
+          echo "agent_version=${AGENT_VERSION}" >> "$GITHUB_OUTPUT"
+
       # Pin QEMU to a known-good version. The default (binfmt:latest) has broken
       # arm64 emulation multiple times due to QEMU segfaults in libc-bin triggers:
       #   - Feb 2025: qemu-v9.2.0 — PR #571 pinned, PR #581 reverted to :latest
@@ -76,7 +85,7 @@ jobs:
           VERSION: ${{ github.event.inputs.tag }}
           SERVERLESS_INIT: true
           ALPINE: ${{ matrix.arrays.isAlpine }}
-          AGENT_VERSION: ${{ github.event.inputs.agentVersion }}
+          AGENT_VERSION: ${{ steps.meta.outputs.agent_version }}
 
       - name: Set up build directory and copy binaries
         run: |

.gitignore

@@ -30,3 +30,4 @@ integration-tests/cdk.context.json
 
 .gitlab/pipeline*
 /CLAUDE.md
+/AGENTS.md

.gitlab/datasources/test-suites.yaml

@@ -3,3 +3,4 @@ test_suites:
   - name: otlp
   - name: snapstart
   - name: lmi
+  - name: auth

.gitlab/scripts/get_secrets.sh

@@ -21,39 +21,15 @@ fi
 
 printf "Getting AWS External ID...\n"
 
-EXTERNAL_ID=$(aws ssm get-parameter \
-    --region us-east-1 \
-    --name "ci.datadog-lambda-extension.$EXTERNAL_ID_NAME" \
-    --with-decryption \
-    --query "Parameter.Value" \
-    --out text)
+EXTERNAL_ID=$(vault kv get -field="$EXTERNAL_ID_NAME" kv/k8s/gitlab-runner/datadog-lambda-extension/secrets)
 
 printf "Getting DD API KEY...\n"
 
-export DD_API_KEY=$(aws ssm get-parameter \
-    --region us-east-1 \
-    --name ci.datadog-lambda-extension.dd-api-key \
-    --with-decryption \
-    --query "Parameter.Value" \
-    --out text)
-
-printf "Getting DD API KEY Secret ARN...\n"
-
-export DATADOG_API_SECRET_ARN=$(aws ssm get-parameter \
-    --region us-east-1 \
-    --name ci.datadog-lambda-extension.dd-api-key-secret-arn \
-    --with-decryption \
-    --query "Parameter.Value" \
-    --out text)
+export DD_API_KEY=$(vault kv get -field=dd-api-key kv/k8s/gitlab-runner/datadog-lambda-extension/secrets)
 
 printf "Getting DD APP KEY...\n"
 
-export DD_APP_KEY=$(aws ssm get-parameter \
-    --region us-east-1 \
-    --name ci.datadog-lambda-extension.dd-app-key \
-    --with-decryption \
-    --query "Parameter.Value" \
-    --out text)
+export DD_APP_KEY=$(vault kv get -field=dd-app-key kv/k8s/gitlab-runner/datadog-lambda-extension/secrets)
 
 printf "Assuming role...\n"
 

.gitlab/templates/pipeline.yaml.tpl

@@ -508,7 +508,7 @@ integration-suite:
     - export CDK_DEFAULT_ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
     - export CDK_DEFAULT_REGION=us-east-1
     - npm run build
-    - npx cdk deploy "integ-${IDENTIFIER}-${TEST_SUITE}" --require-approval never
+    - npx cdk deploy "integ-${IDENTIFIER}-${TEST_SUITE}" --require-approval never --import-existing-resources
     - echo "Running ${TEST_SUITE} integration tests with identifier ${IDENTIFIER}..."
     - export TEST_SUITE=${TEST_SUITE}
     - npx jest tests/${TEST_SUITE}.test.ts

README.md

@@ -21,7 +21,7 @@ For `v67` through `v87`, you can opt out of the next-generation Lambda Extension
 
 Today, all workloads using Logs and Metrics are supported.
 
-APM Tracing is supported for Python, NodeJS, Go, Java, and .NET runtimes.
+APM Tracing is supported for Python, NodeJS, Go, Java, .NET, and Ruby runtimes.
 
 ### Feedback
 

bottlecap/src/bin/bottlecap/main.rs

@@ -51,6 +51,7 @@ use bottlecap::{
             AggregatorHandle as LogsAggregatorHandle, AggregatorService as LogsAggregatorService,
         },
         flusher::LogsFlusher,
+        lambda::DurableContextUpdate,
     },
     otlp::{agent::Agent as OtlpAgent, should_enable_otlp_agent},
     proxy::{interceptor, should_start_proxy},
@@ -149,7 +150,11 @@ async fn main() -> anyhow::Result<()> {
     let config = Arc::new(config::get_config(Path::new(&lambda_directory)));
 
     let aws_config = Arc::new(aws_config);
-    let api_key_factory = create_api_key_factory(&config, &aws_config);
+    // Build one shared reqwest::Client for metrics, logs, trace proxy flushing, and calls to
+    // Datadog APIs (e.g. delegated auth). reqwest::Client is Arc-based internally, so cloning
+    // just increments a refcount and shares the connection pool.
+    let shared_client = bottlecap::http::get_client(&config);
+    let api_key_factory = create_api_key_factory(&config, &aws_config, &shared_client);
 
     let r = response
         .await
@@ -160,6 +165,7 @@ async fn main() -> anyhow::Result<()> {
         Arc::clone(&aws_config),
         &config,
         &client,
+        shared_client,
         &r,
         Arc::clone(&api_key_factory),
         start_time,
@@ -245,17 +251,23 @@ fn get_flush_strategy_for_mode(
     }
 }
 
-fn create_api_key_factory(config: &Arc<Config>, aws_config: &Arc<AwsConfig>) -> Arc<ApiKeyFactory> {
+fn create_api_key_factory(
+    config: &Arc<Config>,
+    aws_config: &Arc<AwsConfig>,
+    client: &reqwest::Client,
+) -> Arc<ApiKeyFactory> {
     let config = Arc::clone(config);
     let aws_config = Arc::clone(aws_config);
+    let client = client.clone();
     let api_key_secret_reload_interval = config.api_key_secret_reload_interval;
 
     Arc::new(ApiKeyFactory::new_from_resolver(
         Arc::new(move || {
             let config = Arc::clone(&config);
             let aws_config = Arc::clone(&aws_config);
+            let client = client.clone();
 
-            Box::pin(async move { resolve_secrets(config, aws_config).await })
+            Box::pin(async move { resolve_secrets(config, aws_config, client).await })
         }),
         api_key_secret_reload_interval,
     ))
@@ -284,6 +296,7 @@ async fn extension_loop_active(
     aws_config: Arc<AwsConfig>,
     config: &Arc<Config>,
     client: &Client,
+    shared_client: reqwest::Client,
     r: &RegisterResponse,
     api_key_factory: Arc<ApiKeyFactory>,
     start_time: Instant,
@@ -293,20 +306,20 @@ async fn extension_loop_active(
     let account_id = r.account_id.as_ref().unwrap_or(&"none".to_string()).clone();
     let tags_provider = setup_tag_provider(&Arc::clone(&aws_config), config, &account_id);
 
-    // Build one shared reqwest::Client for metrics, logs, and trace proxy flushing.
-    // reqwest::Client is Arc-based internally, so cloning just increments a refcount
-    // and shares the connection pool.
-    let shared_client = bottlecap::http::get_client(config);
-
-    let (logs_agent_channel, logs_flusher, logs_agent_cancel_token, logs_aggregator_handle) =
-        start_logs_agent(
-            config,
-            Arc::clone(&api_key_factory),
-            &tags_provider,
-            event_bus_tx.clone(),
-            aws_config.is_managed_instance_mode(),
-            &shared_client,
-        );
+    let (
+        logs_agent_channel,
+        logs_flusher,
+        logs_agent_cancel_token,
+        logs_aggregator_handle,
+        durable_context_tx,
+    ) = start_logs_agent(
+        config,
+        Arc::clone(&api_key_factory),
+        &tags_provider,
+        event_bus_tx.clone(),
+        aws_config.is_managed_instance_mode(),
+        &shared_client,
+    );
 
     let (metrics_flushers, metrics_aggregator_handle, dogstatsd_cancel_token) = start_dogstatsd(
         tags_provider.clone(),
@@ -325,6 +338,7 @@ async fn extension_loop_active(
             Arc::clone(&aws_config),
             metrics_aggregator_handle.clone(),
             Arc::clone(&propagator),
+            durable_context_tx,
         );
     tokio::spawn(async move {
         invocation_processor_service.run().await;
@@ -1039,14 +1053,15 @@ fn start_logs_agent(
     LogsFlusher,
     CancellationToken,
     LogsAggregatorHandle,
+    Sender<DurableContextUpdate>,
 ) {
     let (aggregator_service, aggregator_handle) = LogsAggregatorService::default();
     // Start service in background
     tokio::spawn(async move {
         aggregator_service.run().await;
     });
 
-    let (mut agent, tx) = LogsAgent::new(
+    let (mut agent, tx, durable_context_tx) = LogsAgent::new(
         Arc::clone(tags_provider),
         Arc::clone(config),
         event_bus,
@@ -1068,7 +1083,13 @@ fn start_logs_agent(
         config.clone(),
         client.clone(),
     );
-    (tx, flusher, cancel_token, aggregator_handle)
+    (
+        tx,
+        flusher,
+        cancel_token,
+        aggregator_handle,
+        durable_context_tx,
+    )
 }
 
 #[allow(clippy::type_complexity)]

bottlecap/src/config/env.rs

@@ -482,6 +482,12 @@ pub struct EnvConfig {
     /// The delay between two samples of the API Security schema collection, in seconds.
     #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")]
     pub api_security_sample_delay: Option<Duration>,
+
+    /// @env `DD_ORG_UUID`
+    ///
+    /// The Datadog organization UUID. When set, delegated auth is auto-enabled.
+    #[serde(deserialize_with = "deserialize_string_or_int")]
+    pub org_uuid: Option<String>,
 }
 
 #[allow(clippy::too_many_lines)]
@@ -684,6 +690,8 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) {
     merge_option_to_value!(config, env_config, appsec_waf_timeout);
     merge_option_to_value!(config, env_config, api_security_enabled);
     merge_option_to_value!(config, env_config, api_security_sample_delay);
+
+    merge_string!(config, dd_org_uuid, env_config, org_uuid);
 }
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -1044,6 +1052,8 @@ mod tests {
                 appsec_waf_timeout: Duration::from_secs(1),
                 api_security_enabled: false,
                 api_security_sample_delay: Duration::from_secs(60),
+
+                dd_org_uuid: String::default(),
             };
 
             assert_eq!(config, expected_config);

bottlecap/src/config/mod.rs

@@ -364,6 +364,8 @@ pub struct Config {
     pub span_dedup_timeout: Option<Duration>,
     pub api_key_secret_reload_interval: Option<Duration>,
 
+    pub dd_org_uuid: String,
+
     pub serverless_appsec_enabled: bool,
     pub appsec_rules: Option<String>,
     pub appsec_waf_timeout: Duration,
@@ -479,6 +481,8 @@ impl Default for Config {
             span_dedup_timeout: None,
             api_key_secret_reload_interval: None,
 
+            dd_org_uuid: String::default(),
+
             serverless_appsec_enabled: false,
             appsec_rules: None,
             appsec_waf_timeout: Duration::from_millis(5),