fix(aws-lambda): force placeholder resource so the Lambda Extension drops dd-tracer-serverless-span (#11313)

fix(aws-lambda): force placeholder resource so the Lambda Extension drops dd-tracer-serverless-span

The Datadog Lambda Extension filters the placeholder invocation span
by comparing `span.resource == "dd-tracer-serverless-span"` (see
`bottlecap/src/traces/trace_processor.rs::filter_span_from_lambda_library_or_runtime`).
The intent is that the extension drops the placeholder so the
inferred `aws.lambda` span (which the end-invocation handshake gives
the same `span_id`) is the surviving record under that key in the
trace store and remains parented to the inferred apigateway root.

In practice the HTTP/JAX-RS instrumentation overwrites the
placeholder's resource with the request route (e.g. "POST /") at
`HTTP_FRAMEWORK_ROUTE` priority during the invocation, so the
extension's filter no longer matches at end-of-invocation. Both
records (the placeholder with `parent_id=0`, and the inferred
`aws.lambda` with `parent_id=apigateway.span_id`) reach the backend
under the same `(trace_id, span_id)` key. The trace store keeps the
placeholder, and the rest of the trace is detached from the inferred
apigateway root.

Fix: in `LambdaHandlerInstrumentation.exit`, force the resource name
back to the literal placeholder marker right before `span.finish()`,
using `ResourceNamePriorities.MANUAL_INSTRUMENTATION` so the override
beats whatever HTTP-framework priority the in-flight instrumentation
has written. Verified end-to-end on Quarkus 3.15.4 / Java 21 / Lambda
Extension v96: pre-fix the placeholder leaks to the backend with
`parent_id=0` and the trace store reports orphans; post-fix zero
`dd_tracer_serverless_span` rows reach the backend and the trace
store reports a single 11-span chunk with no orphans.

Refs: SLES-2837

use tag interceptor priority to avoid clobbering user-set resource names

Add unit test

comment

Co-authored-by: rithikanarayan <rithika.narayan@datadoghq.com>
Co-authored-by: devflow.devflow-routing-intake <devflow.devflow-routing-intake@kubernetes.us1.ddbuild.io>

Author: 3 people

dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/main/java/datadog/trace/instrumentation/aws/v1/lambda/LambdaHandlerInstrumentation.java

@@ -24,6 +24,7 @@
 import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
 import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
 import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes;
+import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities;
 import datadog.trace.config.inversion.ConfigHelper;
 import net.bytebuddy.asm.Advice;
 import net.bytebuddy.description.type.TypeDescription;
@@ -126,6 +127,21 @@ static void exit(
         String lambdaRequestId = awsContext.getAwsRequestId();
 
         AgentTracer.get().notifyAppSecEnd(span);
+        // Force the resource name back to the literal placeholder marker right
+        // before finish so that the Datadog Lambda Extension's filter
+        // (filter_span_from_lambda_library_or_runtime in
+        // bottlecap/src/traces/trace_processor.rs, which compares
+        // span.resource == "dd-tracer-serverless-span") drops the placeholder.
+        // Other instrumentation (HTTP/JAX-RS) may have overwritten it with the
+        // route ("POST /") during the invocation, in which case the extension
+        // would fail to dedup, leading to the placeholder leaking to the backend
+        // with parent_id=0 and detaching the inferred apigateway root from the
+        // rest of the trace.
+        // Use TAG_INTERCEPTOR priority because DDSpanContext.setResourceName
+        // ignores writes whose priority is below the current resource priority,
+        // and the HTTP/JAX-RS instrumentation will already have written
+        // HTTP_FRAMEWORK_ROUTE (3) by this point.
+        span.setResourceName(INVOCATION_SPAN_NAME, ResourceNamePriorities.TAG_INTERCEPTOR);
         span.finish();
         AgentTracer.get().notifyExtensionEnd(span, result, null != throwable, lambdaRequestId);
       } finally {

dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/HandlerStreamingSimulatesHttpFrameworkResource.java

@@ -0,0 +1,26 @@
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
+import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
+import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Simulates HTTP server instrumentation updating the local root span resource (e.g. route) while
+ * the Lambda invocation span is active.
+ */
+public class HandlerStreamingSimulatesHttpFrameworkResource implements RequestStreamHandler {
+
+  @Override
+  public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context)
+      throws IOException {
+    AgentSpan span = AgentTracer.activeSpan();
+    if (span != null) {
+      span.setResourceName("POST /api/simulated", ResourceNamePriorities.HTTP_FRAMEWORK_ROUTE);
+    }
+    outputStream.write('O');
+    outputStream.write('K');
+  }
+}

dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/src/test/groovy/LambdaHandlerInstrumentationTest.groovy

@@ -98,6 +98,28 @@ abstract class LambdaHandlerInstrumentationTest extends VersionedNamingTestBase
     }
   }
 
+  def "serverless invocation span resource reset after simulated HTTP framework overwrite"() {
+    when:
+    def input = new ByteArrayInputStream(StandardCharsets.UTF_8.encode("Hello").array())
+    def output = new ByteArrayOutputStream()
+    def ctx = Stub(Context) {
+      getAwsRequestId() >> requestId
+    }
+    new HandlerStreamingSimulatesHttpFrameworkResource().handleRequest(input, output, ctx)
+
+    then:
+    assertTraces(1) {
+      trace(1) {
+        span {
+          operationName operation()
+          resourceName operation()
+          spanType DDSpanTypes.SERVERLESS
+          errored false
+        }
+      }
+    }
+  }
+
   def "test streaming handler with error"() {
     when:
     def input = new ByteArrayInputStream(StandardCharsets.UTF_8.encode("Hello").array())