From 190b609aaff88b26b5662afed33e6818a8f4b726 Mon Sep 17 00:00:00 2001 From: Parker Hyland Date: Fri, 19 Jun 2026 12:18:06 +0200 Subject: [PATCH 1/2] fix: honor faas.invocation_id from non-Lambda instrumentation scopes Spans created from a custom ActivitySource/Tracer (a scope not in the recognized AWS Lambda instrumentation list) had their faas.invocation_id ignored: process_trace_request only extracted the invocation id from spans in recognized Lambda scopes. When no id is found and the Runtime API proxy fallback is unavailable, the receiver discards the trace ("/v1/traces has no invocation IDs, discarding trace"). Extract faas.invocation_id from spans in any scope for invocation correlation. The handler-span rewrites (rename, kind, reparent) stay limited to recognized Lambda scopes, so non-Lambda spans are correlated but otherwise left untouched. Co-Authored-By: Claude Opus 4.8 --- src/otlp/span_mutations.rs | 61 ++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/src/otlp/span_mutations.rs b/src/otlp/span_mutations.rs index fc3a03f..261e833 100644 --- a/src/otlp/span_mutations.rs +++ b/src/otlp/span_mutations.rs @@ -644,6 +644,19 @@ pub fn process_trace_request( for span in &mut scope_span.spans { add_resource_attributes(span); extract_http_body_logs(span); + // Honor an explicitly set faas.invocation_id even when the span + // comes from a non-Lambda instrumentation scope (e.g. a custom + // ActivitySource/Tracer used for manual instrumentation). This is + // used only to correlate the trace to an invocation; the + // handler-span rewrites (rename, kind, reparent) below stay + // limited to recognized Lambda instrumentation scopes. Without + // this, a manually set faas.invocation_id is ignored and the + // trace can be discarded with "no invocation IDs". + if let Some(invocation_id) = extract_invocation_id(span) { + if !invocation_ids.contains(&invocation_id) { + invocation_ids.push(invocation_id); + } + } } continue; } @@ -670,7 +683,7 @@ pub fn process_trace_request( #[cfg(test)] mod tests { - use super::{build_synthetic_trace, StatusCode}; + use super::{build_synthetic_trace, SpanKind, StatusCode}; use crate::state::invocation_data::StoredTrace; use crate::state::invocation_entry; use hyper::{header, Method}; @@ -987,12 +1000,48 @@ mod tests { #[test] #[serial] - fn process_trace_request_ignores_non_lambda_scopes() { - let invocation_id = "inv-process-6"; + fn process_trace_request_honors_invocation_id_in_non_lambda_scope_without_rewrite() { + // A handler span created from a custom ActivitySource (scope + // ".Handler") that manually sets faas.invocation_id. The scope is + // not a recognized Lambda instrumentation scope, but the invocation id must + // still be honored so the trace is correlated and kept rather than discarded. + let invocation_id = "8f3c1d2e-aws-request-id"; store_event_payload(invocation_id, r#"{"test":"data"}"#); - let span = make_span_with_invocation(invocation_id); - let mut request = make_request_with_scope("other.instrumentation.scope", span); + let mut span = make_span_with_invocation(invocation_id); + span.name = "GET /orders".to_string(); + span.kind = SpanKind::Server as i32; + let mut request = make_request_with_scope("payments-api.Handler", span); + let mut invocation_ids = Vec::new(); + let mut encoded_body = Vec::new(); + + super::process_trace_request(&mut request, &mut invocation_ids, &mut encoded_body); + + // The invocation id is extracted even though the scope is not a Lambda scope. + assert_eq!(invocation_ids, vec![invocation_id.to_string()]); + + // ...but the span itself is left untouched: handler-span rewrites stay + // limited to recognized Lambda instrumentation scopes. + let out = &request.resource_spans[0].scope_spans[0].spans[0]; + assert_eq!( + out.name, "GET /orders", + "non-lambda span must not be renamed" + ); + assert_eq!( + out.kind, + SpanKind::Server as i32, + "non-lambda span kind must be preserved" + ); + } + + #[test] + #[serial] + fn process_trace_request_non_lambda_span_without_invocation_id_adds_no_ids() { + let span = Span { + name: "GET /".to_string(), + ..Default::default() + }; + let mut request = make_request_with_scope("System.Net.Http", span); let mut invocation_ids = Vec::new(); let mut encoded_body = Vec::new(); @@ -1000,7 +1049,7 @@ mod tests { assert!( invocation_ids.is_empty(), - "invocation_ids should remain empty for non-lambda scopes" + "a non-lambda span without faas.invocation_id contributes no invocation ids" ); } From 8c27540d6e69f3641bbeb5aa48aa62f4e7241dd0 Mon Sep 17 00:00:00 2001 From: Parker Hyland Date: Fri, 19 Jun 2026 12:18:06 +0200 Subject: [PATCH 2/2] docs: require AWS_LAMBDA_EXEC_WRAPPER for manual instrumentation The Manual Instrumentation steps did not mention AWS_LAMBDA_EXEC_WRAPPER=/opt/wrapper, which is required to enable tracing (it routes the Runtime API through the extension so spans can be correlated to the current invocation and enriched). Without it, traces are discarded with "no invocation IDs" unless every exported span already carries a faas.invocation_id. Add it as an explicit step. Co-Authored-By: Claude Opus 4.8 --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 56b5476..659e864 100644 --- a/README.md +++ b/README.md @@ -125,9 +125,10 @@ The following environment variables allow fine-grained control over secret maski If you prefer to set up OpenTelemetry instrumentation yourself instead of relying on the extension's auto-instrumentation, you can use the manual layer and point your OTLP exporters to the extension's local endpoint. The extension will receive the telemetry, enrich it, and forward it to Dash0. 1. Add the manual layer to your Lambda function: `arn:aws:lambda::115813213817:layer:dash0-extension-manual:`. -2. Configure your OTLP trace exporter to send to `http://127.0.0.1:9009/v1/traces`. -3. If exporting metrics, configure your OTLP metric exporter to send to `http://127.0.0.1:9009/v1/metrics`. -4. Make sure to flush all telemetry before the Lambda invocation completes (e.g., in a response hook or before returning the response). +2. Set `AWS_LAMBDA_EXEC_WRAPPER=/opt/wrapper`. This is **required** (see [Required](#required)) for the extension to correlate spans to invocations and enrich them (trigger, payloads, cold-start/overhead metrics). Without it, traces are discarded with `[DASH0] /v1/traces has no invocation IDs, discarding trace` unless every exported span already carries a `faas.invocation_id` attribute. +3. Configure your OTLP trace exporter to send to `http://127.0.0.1:9009/v1/traces`. +4. If exporting metrics, configure your OTLP metric exporter to send to `http://127.0.0.1:9009/v1/metrics`. +5. Make sure to flush all telemetry before the Lambda invocation completes (e.g., in a response hook or before returning the response). ## Enrichment Attributes