feat(otel): Add X-Ray e2e integration tests for span validation

Author: ayushiahjolia

.github/workflows/delete-stack.yml

@@ -0,0 +1,103 @@
+name: Delete CFN Stack
+
+on:
+  workflow_dispatch:
+    inputs:
+      java_version:
+        description: 'Java version for the stack to delete (17, 21, or 25)'
+        required: true
+        default: '17'
+        type: choice
+        options:
+          - '17'
+          - '21'
+          - '25'
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  delete-stack:
+    env:
+      AWS_REGION: us-west-2
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: "${{ secrets.ACTIONS_INTEGRATION_ROLE_NAME }}"
+          role-session-name: java-language-sdk-delete-stack
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Delete CloudFormation stack
+        run: |
+          STACK_NAME="JavaSDKCloudBasedIntegrationTestStack-Java${{ inputs.java_version }}Runtime"
+          echo "Checking stack status..."
+          STATUS=$(aws cloudformation describe-stacks --stack-name "${STACK_NAME}" --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "NOT_FOUND")
+          echo "Stack status: ${STATUS}"
+
+          if [ "${STATUS}" = "NOT_FOUND" ]; then
+            echo "Stack does not exist, nothing to delete"
+            exit 0
+          fi
+
+          # Handle UPDATE_ROLLBACK_FAILED - need to continue rollback first
+          if [ "${STATUS}" = "UPDATE_ROLLBACK_FAILED" ]; then
+            echo "Stack is in UPDATE_ROLLBACK_FAILED state, continuing rollback..."
+            FAILED_RESOURCES=$(aws cloudformation describe-stack-resources --stack-name "${STACK_NAME}" \
+              --query "StackResources[?ResourceStatus=='UPDATE_FAILED'].LogicalResourceId" --output text)
+            echo "Skipping failed resources: ${FAILED_RESOURCES}"
+            SKIP_ARGS=""
+            for r in ${FAILED_RESOURCES}; do
+              SKIP_ARGS="${SKIP_ARGS} ${r}"
+            done
+            aws cloudformation continue-update-rollback --stack-name "${STACK_NAME}" --resources-to-skip ${SKIP_ARGS}
+            echo "Waiting for rollback to complete..."
+            aws cloudformation wait stack-rollback-complete --stack-name "${STACK_NAME}"
+            echo "Rollback complete, proceeding to delete..."
+          fi
+
+          # Handle UPDATE_FAILED - need to continue rollback first
+          if [ "${STATUS}" = "UPDATE_FAILED" ]; then
+            echo "Stack is in UPDATE_FAILED state, continuing rollback..."
+            FAILED_RESOURCES=$(aws cloudformation describe-stack-resources --stack-name "${STACK_NAME}" \
+              --query "StackResources[?ResourceStatus=='UPDATE_FAILED'].LogicalResourceId" --output text)
+            echo "Skipping failed resources: ${FAILED_RESOURCES}"
+            SKIP_ARGS=""
+            for r in ${FAILED_RESOURCES}; do
+              SKIP_ARGS="${SKIP_ARGS} ${r}"
+            done
+            aws cloudformation continue-update-rollback --stack-name "${STACK_NAME}" --resources-to-skip ${SKIP_ARGS}
+            echo "Waiting for rollback to complete..."
+            aws cloudformation wait stack-rollback-complete --stack-name "${STACK_NAME}"
+            echo "Rollback complete, proceeding to delete..."
+          fi
+
+          # Refresh status after potential rollback
+          STATUS=$(aws cloudformation describe-stacks --stack-name "${STACK_NAME}" --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "NOT_FOUND")
+          echo "Current stack status: ${STATUS}"
+
+          if [ "${STATUS}" = "NOT_FOUND" ]; then
+            echo "Stack no longer exists"
+            exit 0
+          fi
+
+          # Handle DELETE_FAILED - retry with retain
+          if [ "${STATUS}" = "DELETE_FAILED" ]; then
+            echo "Stack is in DELETE_FAILED state, retrying delete with --retain-resources..."
+            FAILED_RESOURCES=$(aws cloudformation describe-stack-resources --stack-name "${STACK_NAME}" \
+              --query "StackResources[?ResourceStatus=='DELETE_FAILED'].LogicalResourceId" --output text)
+            echo "Retaining failed resources: ${FAILED_RESOURCES}"
+            RETAIN_ARGS=""
+            for r in ${FAILED_RESOURCES}; do
+              RETAIN_ARGS="${RETAIN_ARGS} ${r}"
+            done
+            aws cloudformation delete-stack --stack-name "${STACK_NAME}" --retain-resources ${RETAIN_ARGS}
+          else
+            echo "Deleting stack: ${STACK_NAME}"
+            aws cloudformation delete-stack --stack-name "${STACK_NAME}"
+          fi
+
+          echo "Waiting for stack deletion to complete..."
+          aws cloudformation wait stack-delete-complete --stack-name "${STACK_NAME}"
+          echo "Stack ${STACK_NAME} deleted successfully"

.github/workflows/e2e-tests.yml

@@ -71,6 +71,45 @@ jobs:
           sam build --debug --parameter-overrides \
             'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=RoleArn,ParameterValue=${{ secrets.DURABLE_INTEGRATION_TEST_ROLE_ARN }}'
         working-directory: ./examples
+      - name: Clean up stuck stack if needed
+        run: |
+          STACK_NAME="JavaSDKCloudBasedIntegrationTestStack-Java${{ matrix.java }}Runtime"
+          STATUS=$(aws cloudformation describe-stacks --stack-name "${STACK_NAME}" --query 'Stacks[0].StackStatus' --output text 2>/dev/null || echo "NOT_FOUND")
+          echo "Stack status: ${STATUS}"
+
+          if [[ "${STATUS}" == "UPDATE_FAILED" || "${STATUS}" == "UPDATE_ROLLBACK_FAILED" ]]; then
+            echo "Stack is in ${STATUS} state, attempting rollback..."
+            # Get resources that failed to update
+            FAILED_RESOURCES=$(aws cloudformation describe-stack-resources --stack-name "${STACK_NAME}" \
+              --query "StackResources[?ResourceStatus=='UPDATE_FAILED'].LogicalResourceId" --output text)
+            echo "Failed resources: ${FAILED_RESOURCES}"
+            SKIP_ARGS=""
+            for r in ${FAILED_RESOURCES}; do
+              SKIP_ARGS="${SKIP_ARGS} ${r}"
+            done
+            if [ -n "${SKIP_ARGS}" ]; then
+              aws cloudformation continue-update-rollback --stack-name "${STACK_NAME}" --resources-to-skip ${SKIP_ARGS}
+            else
+              aws cloudformation continue-update-rollback --stack-name "${STACK_NAME}"
+            fi
+            echo "Waiting for rollback to complete..."
+            aws cloudformation wait stack-rollback-complete --stack-name "${STACK_NAME}"
+            echo "Rollback complete, deleting stack..."
+            aws cloudformation delete-stack --stack-name "${STACK_NAME}"
+            aws cloudformation wait stack-delete-complete --stack-name "${STACK_NAME}"
+            echo "Stack deleted, SAM will create fresh"
+          elif [ "${STATUS}" = "DELETE_FAILED" ]; then
+            echo "Stack is in DELETE_FAILED state, retrying with retain..."
+            FAILED_RESOURCES=$(aws cloudformation describe-stack-resources --stack-name "${STACK_NAME}" \
+              --query "StackResources[?ResourceStatus=='DELETE_FAILED'].LogicalResourceId" --output text)
+            RETAIN_ARGS=""
+            for r in ${FAILED_RESOURCES}; do
+              RETAIN_ARGS="${RETAIN_ARGS} ${r}"
+            done
+            aws cloudformation delete-stack --stack-name "${STACK_NAME}" --retain-resources ${RETAIN_ARGS}
+            aws cloudformation wait stack-delete-complete --stack-name "${STACK_NAME}"
+            echo "Stack deleted, SAM will create fresh"
+          fi
       - name: sam deploy
         run: | 
           sam deploy --stack-name JavaSDKCloudBasedIntegrationTestStack-Java${{ matrix.java }}Runtime \

examples/pom.xml

@@ -50,6 +50,18 @@
             <version>1.63.0</version>
         </dependency>
 
+        <!-- OTLP gRPC exporter (required for X-Ray via ADOT collector) -->
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-otlp</artifactId>
+            <version>1.63.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-netty-shaded</artifactId>
+            <version>1.72.0</version>
+        </dependency>
+
         <!-- AWS Lambda Java Core -->
         <dependency>
             <groupId>com.amazonaws</groupId>
@@ -91,6 +103,11 @@
             <artifactId>sts</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>xray</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayStepExample.java

@@ -0,0 +1,55 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.examples.otel;
+
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.DurableHandler;
+import software.amazon.lambda.durable.examples.types.GreetingRequest;
+import software.amazon.lambda.durable.otel.OpenTelemetryDurablePlugin;
+
+/**
+ * OTel + X-Ray example: simple steps in a single invocation.
+ *
+ * <p>Exports spans via OTLP gRPC to the ADOT collector extension (Lambda layer), which forwards to X-Ray. Used by
+ * {@code OtelXRayIntegrationTest} to verify spans appear correctly in X-Ray.
+ *
+ * <p>Expected trace structure in X-Ray:
+ *
+ * <pre>
+ * durable.invocation
+ * ├── durable.step:create-greeting
+ * │   └── durable.step:create-greeting [attempt 1]
+ * └── durable.step:transform
+ *     └── durable.step:transform [attempt 1]
+ * </pre>
+ */
+public class OtelXRayStepExample extends DurableHandler<GreetingRequest, String> {
+
+    @Override
+    protected DurableConfig createConfiguration() {
+        var otlpExporter = OtlpGrpcSpanExporter.builder()
+                .setEndpoint("http://localhost:4317")
+                .build();
+
+        var otelPlugin = new OpenTelemetryDurablePlugin(
+                SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)));
+
+        return DurableConfig.builder().withPlugins(otelPlugin).build();
+    }
+
+    @Override
+    public String handleRequest(GreetingRequest input, DurableContext context) {
+        context.getLogger().info("Starting OTel X-Ray step example for {}", input.getName());
+
+        var greeting = context.step("create-greeting", String.class, stepCtx -> "Hello, " + input.getName());
+
+        var result = context.step("transform", String.class, stepCtx -> greeting.toUpperCase() + "!");
+
+        context.getLogger().info("OTel X-Ray step example complete: {}", result);
+        return result;
+    }
+}

examples/src/main/java/software/amazon/lambda/durable/examples/otel/OtelXRayWaitExample.java

@@ -0,0 +1,74 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package software.amazon.lambda.durable.examples.otel;
+
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import java.time.Duration;
+import software.amazon.lambda.durable.DurableConfig;
+import software.amazon.lambda.durable.DurableContext;
+import software.amazon.lambda.durable.DurableHandler;
+import software.amazon.lambda.durable.examples.types.GreetingRequest;
+import software.amazon.lambda.durable.otel.OpenTelemetryDurablePlugin;
+
+/**
+ * OTel + X-Ray example: step → wait → step pattern that forces multiple Lambda invocations.
+ *
+ * <p>This handler exercises the critical multi-invocation tracing scenario:
+ *
+ * <ol>
+ *   <li>Invocation 1: "before-wait" step completes → wait suspends execution
+ *   <li>Invocation 2: replays "before-wait" (no-op) → wait completes → "after-wait" step runs
+ * </ol>
+ *
+ * <p>Exports spans via OTLP gRPC to the ADOT collector extension (Lambda layer), which forwards to X-Ray.
+ *
+ * <p>Used by {@code OtelXRayIntegrationTest} to verify that deterministic trace IDs correctly stitch spans from
+ * multiple invocations into a single X-Ray trace.
+ *
+ * <p>Expected trace structure in X-Ray:
+ *
+ * <pre>
+ * Trace (single trace ID across both invocations)
+ * ├── durable.invocation (invocation 1)
+ * │   ├── durable.step:before-wait
+ * │   │   └── durable.step:before-wait [attempt 1]
+ * │   └── durable.wait:pause (ended as PENDING)
+ * └── durable.invocation (invocation 2)
+ *     ├── durable.wait:pause (completed)
+ *     └── durable.step:after-wait
+ *         └── durable.step:after-wait [attempt 1]
+ * </pre>
+ *
+ * <p>All spans share the same deterministic trace ID derived from the execution ARN.
+ */
+public class OtelXRayWaitExample extends DurableHandler<GreetingRequest, String> {
+
+    @Override
+    protected DurableConfig createConfiguration() {
+        var otlpExporter = OtlpGrpcSpanExporter.builder()
+                .setEndpoint("http://localhost:4317")
+                .build();
+
+        var otelPlugin = new OpenTelemetryDurablePlugin(
+                SdkTracerProvider.builder().addSpanProcessor(SimpleSpanProcessor.create(otlpExporter)));
+
+        return DurableConfig.builder().withPlugins(otelPlugin).build();
+    }
+
+    @Override
+    public String handleRequest(GreetingRequest input, DurableContext context) {
+        context.getLogger().info("Starting OTel X-Ray wait example for {}", input.getName());
+
+        var before = context.step("before-wait", String.class, stepCtx -> "Prepared: " + input.getName());
+
+        // This wait forces Lambda to suspend and re-invoke after the duration
+        context.wait("pause", Duration.ofSeconds(5));
+
+        var after = context.step("after-wait", String.class, stepCtx -> before + " | Resumed and completed");
+
+        context.getLogger().info("OTel X-Ray wait example complete: {}", after);
+        return after;
+    }
+}

examples/src/main/resources/collector.yaml

@@ -0,0 +1,16 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "localhost:4317"
+      http:
+        endpoint: "localhost:4318"
+
+exporters:
+  awsxray:
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [awsxray]