From e677d7e58a2e33fa9cea03c4974b2fde2ba79431 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Wed, 28 Jan 2026 15:48:46 -0500 Subject: [PATCH 1/4] [APMSVLS-197] feat: Add fallback to extract trace context from event.request.headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automatic trace context extraction from event.request.headers for AppSync integration scenarios. This eliminates the need for customers to use customized extractors for RUM → AppSync → Lambda resolver flows where RUM-injected trace context is nested under event["request"]["headers"]. --- bottlecap/src/lifecycle/invocation/processor.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 7b8079487..28371bdec 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -1088,6 +1088,15 @@ impl Processor { return Some(sc); } + if let Some(request_obj) = payload_value.get("request") { + if let Some(request_headers) = request_obj.get("headers") { + if let Some(sc) = propagator.extract(request_headers) { + debug!("Extracted trace context from event.request.headers"); + return Some(sc); + } + } + } + None } From 50ae32f9210e9e15c59339afdc0a928ea460cec1 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 29 Jan 2026 11:50:55 -0500 Subject: [PATCH 2/4] [APMSVLS-197] test: Add unit tests for event.request.headers trace context extraction --- .../src/lifecycle/invocation/processor.rs | 291 ++++++++++++++++++ 1 file changed, 291 insertions(+) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 28371bdec..148f0762e 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -1860,4 +1860,295 @@ mod tests { "Should be snapstart span (id=3), not cold start span (id=2)" ); } + + #[test] + fn test_extract_span_context_from_event_request_headers_datadog() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "request": { + "headers": { + "x-datadog-trace-id": "1234567890", + "x-datadog-parent-id": "9876543210", + "x-datadog-sampling-priority": "2", + "x-datadog-origin": "rum" + } + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!(result.is_some()); + let context = result.unwrap(); + assert_eq!(context.trace_id, 1_234_567_890); + assert_eq!(context.span_id, 9_876_543_210); + assert_eq!(context.sampling.unwrap().priority, Some(2)); + assert_eq!(context.origin, Some("rum".to_string())); + } + + #[test] + fn test_extract_span_context_from_event_request_headers_tracecontext() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "request": { + "headers": { + "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "tracestate": "dd=s:2;o:rum" + } + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!(result.is_some()); + let context = result.unwrap(); + assert_eq!(context.sampling.unwrap().priority, Some(2)); + assert_eq!(context.origin, Some("rum".to_string())); + } + + #[test] + fn test_extract_span_context_priority_order() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + + let mut headers = HashMap::new(); + headers.insert(DATADOG_TRACE_ID_KEY.to_string(), "111".to_string()); + headers.insert(DATADOG_PARENT_ID_KEY.to_string(), "222".to_string()); + + let payload = json!({ + "headers": { + "x-datadog-trace-id": "333", + "x-datadog-parent-id": "444" + }, + "request": { + "headers": { + "x-datadog-trace-id": "555", + "x-datadog-parent-id": "666" + } + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!(result.is_some()); + let context = result.unwrap(); + assert_eq!( + context.trace_id, 333, + "Should prioritize event.headers over other sources" + ); + } + + #[test] + fn test_extract_span_context_fallback_to_request_headers() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "argumentsMap": { + "id": "123" + }, + "request": { + "headers": { + "x-datadog-trace-id": "7777", + "x-datadog-parent-id": "8888", + "x-datadog-sampling-priority": "1" + } + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!(result.is_some(), "Should fallback to event.request.headers"); + let context = result.unwrap(); + assert_eq!(context.trace_id, 7777); + assert_eq!(context.span_id, 8888); + assert_eq!(context.sampling.unwrap().priority, Some(1)); + } + + #[test] + fn test_extract_span_context_no_request_headers() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "argumentsMap": { + "id": "123" + }, + "request": { + "body": "some body" + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!( + result.is_none(), + "Should return None when no trace context found" + ); + } + + #[test] + fn test_extract_span_context_empty_request_headers() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "request": { + "headers": {} + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!( + result.is_none(), + "Should return None when request.headers is empty" + ); + } + + #[test] + fn test_extract_span_context_invalid_request_headers() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "request": { + "headers": { + "x-datadog-trace-id": "not-a-number", + "x-datadog-parent-id": "also-not-a-number" + } + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!( + result.is_none(), + "Should return None when headers are invalid" + ); + } + + #[test] + fn test_extract_span_context_appsync_real_world_example() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "arguments": { + "userId": "12345" + }, + "identity": { + "sub": "user-sub-id" + }, + "request": { + "headers": { + "x-datadog-trace-id": "123456789012345", + "x-datadog-parent-id": "98765432109876", + "x-datadog-sampling-priority": "2", + "x-datadog-origin": "rum", + "x-datadog-tags": "_dd.p.dm=-0", + "user-agent": "Mozilla/5.0", + "content-type": "application/json" + } + }, + "info": { + "fieldName": "getUser", + "parentTypeName": "Query" + } + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!( + result.is_some(), + "Should extract from real-world AppSync event" + ); + let context = result.unwrap(); + assert_eq!(context.trace_id, 123_456_789_012_345); + assert_eq!(context.span_id, 98_765_432_109_876); + assert_eq!(context.sampling.unwrap().priority, Some(2)); + assert_eq!(context.origin, Some("rum".to_string())); + } + + #[test] + fn test_extract_span_context_request_not_an_object() { + let config = Arc::new(config::Config { + trace_propagation_style_extract: vec![ + config::trace_propagation_style::TracePropagationStyle::Datadog, + config::trace_propagation_style::TracePropagationStyle::TraceContext, + ], + ..config::Config::default() + }); + let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); + let headers = HashMap::new(); + + let payload = json!({ + "request": "invalid-not-an-object" + }); + + let result = Processor::extract_span_context(&headers, &payload, propagator); + + assert!( + result.is_none(), + "Should handle gracefully when request is not an object" + ); + } } From 87af0a47ed3c3a2ec998fbdac10da1a7a1489e95 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 29 Jan 2026 14:39:27 -0500 Subject: [PATCH 3/4] [APMSVLS-197] refactor: Prioritize event.request.headers extraction for service-specific traces --- .../src/lifecycle/invocation/processor.rs | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 148f0762e..a971daba5 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -1076,6 +1076,15 @@ impl Processor { return Some(sc); } + if let Some(request_obj) = payload_value.get("request") { + if let Some(request_headers) = request_obj.get("headers") { + if let Some(sc) = propagator.extract(request_headers) { + debug!("Extracted trace context from event.request.headers"); + return Some(sc); + } + } + } + if let Some(payload_headers) = payload_value.get("headers") { if let Some(sc) = propagator.extract(payload_headers) { debug!("Extracted trace context from event headers"); @@ -1088,15 +1097,6 @@ impl Processor { return Some(sc); } - if let Some(request_obj) = payload_value.get("request") { - if let Some(request_headers) = request_obj.get("headers") { - if let Some(sc) = propagator.extract(request_headers) { - debug!("Extracted trace context from event.request.headers"); - return Some(sc); - } - } - } - None } @@ -1956,8 +1956,8 @@ mod tests { assert!(result.is_some()); let context = result.unwrap(); assert_eq!( - context.trace_id, 333, - "Should prioritize event.headers over other sources" + context.trace_id, 555, + "Should prioritize event.request.headers as service-specific extraction" ); } @@ -1988,7 +1988,10 @@ mod tests { let result = Processor::extract_span_context(&headers, &payload, propagator); - assert!(result.is_some(), "Should fallback to event.request.headers"); + assert!( + result.is_some(), + "Should extract from event.request.headers" + ); let context = result.unwrap(); assert_eq!(context.trace_id, 7777); assert_eq!(context.span_id, 8888); From 3ef4752d96ea10006975ea83d4cf272d442678a0 Mon Sep 17 00:00:00 2001 From: David Ogbureke Date: Thu, 5 Feb 2026 14:29:54 -0500 Subject: [PATCH 4/4] [APMSVLS-197] refactor: Clean up trace context extraction code and tests. Removed redundant tests that would be caught by others. Priority and functionality is entirely covered within test_extract_span_context_priority_order. --- .../src/lifecycle/invocation/processor.rs | 241 +----------------- 1 file changed, 7 insertions(+), 234 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index a971daba5..296eee3d4 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -1076,13 +1076,13 @@ impl Processor { return Some(sc); } - if let Some(request_obj) = payload_value.get("request") { - if let Some(request_headers) = request_obj.get("headers") { - if let Some(sc) = propagator.extract(request_headers) { - debug!("Extracted trace context from event.request.headers"); - return Some(sc); - } - } + if let Some(sc) = payload_value + .get("request") + .and_then(|req| req.get("headers")) + .and_then(|headers| propagator.extract(headers)) + { + debug!("Extracted trace context from event.request.headers"); + return Some(sc); } if let Some(payload_headers) = payload_value.get("headers") { @@ -1861,68 +1861,6 @@ mod tests { ); } - #[test] - fn test_extract_span_context_from_event_request_headers_datadog() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "request": { - "headers": { - "x-datadog-trace-id": "1234567890", - "x-datadog-parent-id": "9876543210", - "x-datadog-sampling-priority": "2", - "x-datadog-origin": "rum" - } - } - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!(result.is_some()); - let context = result.unwrap(); - assert_eq!(context.trace_id, 1_234_567_890); - assert_eq!(context.span_id, 9_876_543_210); - assert_eq!(context.sampling.unwrap().priority, Some(2)); - assert_eq!(context.origin, Some("rum".to_string())); - } - - #[test] - fn test_extract_span_context_from_event_request_headers_tracecontext() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "request": { - "headers": { - "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", - "tracestate": "dd=s:2;o:rum" - } - } - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!(result.is_some()); - let context = result.unwrap(); - assert_eq!(context.sampling.unwrap().priority, Some(2)); - assert_eq!(context.origin, Some("rum".to_string())); - } - #[test] fn test_extract_span_context_priority_order() { let config = Arc::new(config::Config { @@ -1961,43 +1899,6 @@ mod tests { ); } - #[test] - fn test_extract_span_context_fallback_to_request_headers() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "argumentsMap": { - "id": "123" - }, - "request": { - "headers": { - "x-datadog-trace-id": "7777", - "x-datadog-parent-id": "8888", - "x-datadog-sampling-priority": "1" - } - } - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!( - result.is_some(), - "Should extract from event.request.headers" - ); - let context = result.unwrap(); - assert_eq!(context.trace_id, 7777); - assert_eq!(context.span_id, 8888); - assert_eq!(context.sampling.unwrap().priority, Some(1)); - } - #[test] fn test_extract_span_context_no_request_headers() { let config = Arc::new(config::Config { @@ -2026,132 +1927,4 @@ mod tests { "Should return None when no trace context found" ); } - - #[test] - fn test_extract_span_context_empty_request_headers() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "request": { - "headers": {} - } - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!( - result.is_none(), - "Should return None when request.headers is empty" - ); - } - - #[test] - fn test_extract_span_context_invalid_request_headers() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "request": { - "headers": { - "x-datadog-trace-id": "not-a-number", - "x-datadog-parent-id": "also-not-a-number" - } - } - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!( - result.is_none(), - "Should return None when headers are invalid" - ); - } - - #[test] - fn test_extract_span_context_appsync_real_world_example() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "arguments": { - "userId": "12345" - }, - "identity": { - "sub": "user-sub-id" - }, - "request": { - "headers": { - "x-datadog-trace-id": "123456789012345", - "x-datadog-parent-id": "98765432109876", - "x-datadog-sampling-priority": "2", - "x-datadog-origin": "rum", - "x-datadog-tags": "_dd.p.dm=-0", - "user-agent": "Mozilla/5.0", - "content-type": "application/json" - } - }, - "info": { - "fieldName": "getUser", - "parentTypeName": "Query" - } - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!( - result.is_some(), - "Should extract from real-world AppSync event" - ); - let context = result.unwrap(); - assert_eq!(context.trace_id, 123_456_789_012_345); - assert_eq!(context.span_id, 98_765_432_109_876); - assert_eq!(context.sampling.unwrap().priority, Some(2)); - assert_eq!(context.origin, Some("rum".to_string())); - } - - #[test] - fn test_extract_span_context_request_not_an_object() { - let config = Arc::new(config::Config { - trace_propagation_style_extract: vec![ - config::trace_propagation_style::TracePropagationStyle::Datadog, - config::trace_propagation_style::TracePropagationStyle::TraceContext, - ], - ..config::Config::default() - }); - let propagator = Arc::new(DatadogCompositePropagator::new(Arc::clone(&config))); - let headers = HashMap::new(); - - let payload = json!({ - "request": "invalid-not-an-object" - }); - - let result = Processor::extract_span_context(&headers, &payload, propagator); - - assert!( - result.is_none(), - "Should handle gracefully when request is not an object" - ); - } }