diff --git a/.gitlab/config.yaml b/.gitlab/config.yaml index 08b7531b0..d75b9ca92 100644 --- a/.gitlab/config.yaml +++ b/.gitlab/config.yaml @@ -9,9 +9,12 @@ outputFiles: datasources: flavors: url: .gitlab/datasources/flavors.yaml - + environments: url: .gitlab/datasources/environments.yaml regions: url: .gitlab/datasources/regions.yaml + + test_suites: + url: .gitlab/datasources/test-suites.yaml diff --git a/.gitlab/datasources/test-suites.yaml b/.gitlab/datasources/test-suites.yaml new file mode 100644 index 000000000..7eb016808 --- /dev/null +++ b/.gitlab/datasources/test-suites.yaml @@ -0,0 +1,4 @@ +test_suites: + - name: base + - name: otlp + - name: snapstart diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl index d38440d0a..d57f9c758 100644 --- a/.gitlab/templates/pipeline.yaml.tpl +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -466,7 +466,9 @@ integration-suite: image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} parallel: matrix: - - TEST_SUITE: [base, otlp] + - TEST_SUITE: {{ range (ds "test_suites").test_suites }} + - {{ .name }} + {{- end}} rules: - when: on_success needs: diff --git a/integration-tests/bin/app.ts b/integration-tests/bin/app.ts index f1dd9077f..37aec513d 100644 --- a/integration-tests/bin/app.ts +++ b/integration-tests/bin/app.ts @@ -3,6 +3,7 @@ import 'source-map-support/register'; import * as cdk from 'aws-cdk-lib'; import {Base} from '../lib/stacks/base'; import {Otlp} from '../lib/stacks/otlp'; +import {Snapstart} from '../lib/stacks/snapstart'; import {getIdentifier} from '../tests/utils/config'; const app = new cdk.App(); @@ -21,6 +22,9 @@ const stacks = [ new Otlp(app, `integ-${identifier}-otlp`, { env, }), + new Snapstart(app, `integ-${identifier}-snapstart`, { + env, + }), ] // Tag all stacks so we can easily clean them up diff --git a/integration-tests/lambda/snapstart-dotnet/.gitignore b/integration-tests/lambda/snapstart-dotnet/.gitignore new file mode 100644 index 000000000..cd42ee34e --- /dev/null +++ b/integration-tests/lambda/snapstart-dotnet/.gitignore @@ -0,0 +1,2 @@ +bin/ +obj/ diff --git a/integration-tests/lambda/snapstart-dotnet/Function.cs b/integration-tests/lambda/snapstart-dotnet/Function.cs new file mode 100644 index 000000000..8fbaa8a7a --- /dev/null +++ b/integration-tests/lambda/snapstart-dotnet/Function.cs @@ -0,0 +1,23 @@ +using Amazon.Lambda.Core; +using System.Collections.Generic; +using System.Threading.Tasks; + +[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))] + +namespace Function +{ + public class SnapstartHandler + { + public async Task> FunctionHandler(Dictionary input, ILambdaContext context) + { + // Wait 10 seconds to guarantee concurrent execution overlap + await Task.Delay(10000); + + return new Dictionary + { + { "statusCode", 200 }, + { "body", "Snapstart .NET function executed" } + }; + } + } +} diff --git a/integration-tests/lambda/snapstart-dotnet/Function.csproj b/integration-tests/lambda/snapstart-dotnet/Function.csproj new file mode 100644 index 000000000..64fda299c --- /dev/null +++ b/integration-tests/lambda/snapstart-dotnet/Function.csproj @@ -0,0 +1,14 @@ + + + net8.0 + true + Lambda + true + true + + + + + + + diff --git a/integration-tests/lambda/snapstart-java/.gitignore b/integration-tests/lambda/snapstart-java/.gitignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/integration-tests/lambda/snapstart-java/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/integration-tests/lambda/snapstart-java/pom.xml b/integration-tests/lambda/snapstart-java/pom.xml new file mode 100644 index 000000000..28ca0c1d5 --- /dev/null +++ b/integration-tests/lambda/snapstart-java/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + example + snapstart-java-lambda + 1.0.0 + jar + + Snapstart Java Lambda + Snapstart Java Lambda function for Datadog Extension integration testing + + + 21 + 21 + UTF-8 + + + + + com.amazonaws + aws-lambda-java-core + 1.2.3 + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.0 + + + package + + shade + + + function + false + + + + + + + diff --git a/integration-tests/lambda/snapstart-java/src/main/java/example/SnapstartHandler.java b/integration-tests/lambda/snapstart-java/src/main/java/example/SnapstartHandler.java new file mode 100644 index 000000000..9e8fa7bdd --- /dev/null +++ b/integration-tests/lambda/snapstart-java/src/main/java/example/SnapstartHandler.java @@ -0,0 +1,24 @@ +package example; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestHandler; +import java.util.Map; +import java.util.HashMap; + +public class SnapstartHandler implements RequestHandler, Map> { + @Override + public Map handleRequest(Map event, Context context) { + + // Wait 10 seconds to guarantee concurrent execution overlap + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + Map response = new HashMap<>(); + response.put("statusCode", 200); + response.put("body", "Snapstart Java function executed"); + return response; + } +} diff --git a/integration-tests/lambda/snapstart-python/lambda_function.py b/integration-tests/lambda/snapstart-python/lambda_function.py new file mode 100644 index 000000000..c71072601 --- /dev/null +++ b/integration-tests/lambda/snapstart-python/lambda_function.py @@ -0,0 +1,14 @@ +import logging +import time + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +def handler(event, context): + # Wait 10 seconds to guarantee concurrent execution overlap + time.sleep(10) + + return { + 'statusCode': 200, + 'body': 'Snapstart Python function executed' + } diff --git a/integration-tests/lambda/snapstart-python/requirements.txt b/integration-tests/lambda/snapstart-python/requirements.txt new file mode 100644 index 000000000..1e5589009 --- /dev/null +++ b/integration-tests/lambda/snapstart-python/requirements.txt @@ -0,0 +1 @@ +# No additional dependencies required for this simple handler diff --git a/integration-tests/lib/stacks/snapstart.ts b/integration-tests/lib/stacks/snapstart.ts new file mode 100644 index 000000000..e0a8cf0d9 --- /dev/null +++ b/integration-tests/lib/stacks/snapstart.ts @@ -0,0 +1,106 @@ +import * as cdk from 'aws-cdk-lib'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { Construct } from 'constructs'; +import { + createLogGroup, + defaultDatadogEnvVariables, + defaultDatadogSecretPolicy, + getExtensionLayer, + getPython313Layer, + getJava21Layer, + getDotnet8Layer +} from '../util'; + +export class Snapstart extends cdk.Stack { + constructor(scope: Construct, id: string, props: cdk.StackProps) { + super(scope, id, props); + + const extensionLayer = getExtensionLayer(this); + const python313Layer = getPython313Layer(this); + const java21Layer = getJava21Layer(this); + const dotnet8Layer = getDotnet8Layer(this); + + const javaFunctionName = `${id}-java-lambda`; + const javaFunction = new lambda.Function(this, javaFunctionName, { + runtime: lambda.Runtime.JAVA_21, + architecture: lambda.Architecture.ARM_64, + handler: 'example.SnapstartHandler::handleRequest', + code: lambda.Code.fromAsset('./lambda/snapstart-java/target/function.jar'), + functionName: javaFunctionName, + timeout: cdk.Duration.seconds(30), + memorySize: 512, + snapStart: lambda.SnapStartConf.ON_PUBLISHED_VERSIONS, + environment: { + ...defaultDatadogEnvVariables, + DD_SERVICE: javaFunctionName, + AWS_LAMBDA_EXEC_WRAPPER: '/opt/datadog_wrapper', + DD_TRACE_ENABLED: 'true', + }, + logGroup: createLogGroup(this, javaFunctionName) + }); + javaFunction.addToRolePolicy(defaultDatadogSecretPolicy); + javaFunction.addLayers(extensionLayer); + javaFunction.addLayers(java21Layer); + const javaVersion = javaFunction.currentVersion; + const javaAlias = new lambda.Alias(this, `${javaFunctionName}-snapstart-alias`, { + aliasName: 'snapstart', + version: javaVersion, + }); + + const pythonFunctionName = `${id}-python-lambda`; + const pythonFunction = new lambda.Function(this, pythonFunctionName, { + runtime: lambda.Runtime.PYTHON_3_13, + architecture: lambda.Architecture.ARM_64, + handler: 'datadog_lambda.handler.handler', + code: lambda.Code.fromAsset('./lambda/snapstart-python'), + functionName: pythonFunctionName, + timeout: cdk.Duration.seconds(30), + memorySize: 512, + snapStart: lambda.SnapStartConf.ON_PUBLISHED_VERSIONS, + environment: { + ...defaultDatadogEnvVariables, + DD_SERVICE: pythonFunctionName, + DD_TRACE_ENABLED: 'true', + DD_LAMBDA_HANDLER: 'lambda_function.handler', + DD_TRACE_AGENT_URL: 'http://127.0.0.1:8126', + DD_COLD_START_TRACING: 'true', + DD_MIN_COLD_START_DURATION: '0', + }, + logGroup: createLogGroup(this, pythonFunctionName) + }); + pythonFunction.addToRolePolicy(defaultDatadogSecretPolicy); + pythonFunction.addLayers(extensionLayer); + pythonFunction.addLayers(python313Layer); + const pythonVersion = pythonFunction.currentVersion; + const pythonAlias = new lambda.Alias(this, `${pythonFunctionName}-snapstart-alias`, { + aliasName: 'snapstart', + version: pythonVersion, + }); + + const dotnetFunctionName = `${id}-dotnet-lambda`; + const dotnetFunction = new lambda.Function(this, dotnetFunctionName, { + runtime: lambda.Runtime.DOTNET_8, + architecture: lambda.Architecture.ARM_64, + handler: 'Function::Function.SnapstartHandler::FunctionHandler', + code: lambda.Code.fromAsset('./lambda/snapstart-dotnet/bin/function.zip'), + functionName: dotnetFunctionName, + timeout: cdk.Duration.seconds(30), + memorySize: 512, + snapStart: lambda.SnapStartConf.ON_PUBLISHED_VERSIONS, + environment: { + ...defaultDatadogEnvVariables, + DD_SERVICE: dotnetFunctionName, + AWS_LAMBDA_EXEC_WRAPPER: '/opt/datadog_wrapper', + }, + logGroup: createLogGroup(this, dotnetFunctionName) + }); + dotnetFunction.addToRolePolicy(defaultDatadogSecretPolicy); + dotnetFunction.addLayers(extensionLayer); + dotnetFunction.addLayers(dotnet8Layer); + const dotnetVersion = dotnetFunction.currentVersion; + const dotnetAlias = new lambda.Alias(this, `${dotnetFunctionName}-snapstart-alias`, { + aliasName: 'snapstart', + version: dotnetVersion, + }); + } +} diff --git a/integration-tests/tests/snapstart.test.ts b/integration-tests/tests/snapstart.test.ts new file mode 100644 index 000000000..bd0e60919 --- /dev/null +++ b/integration-tests/tests/snapstart.test.ts @@ -0,0 +1,136 @@ +import { invokeLambdaAndGetDatadogData, LambdaInvocationDatadogData } from './utils/util'; +import { getIdentifier } from './utils/config'; +import { publishVersion, waitForSnapStartReady } from './utils/lambda'; + +describe('Snapstart Integration Tests', () => { + const identifier = getIdentifier(); + + const results = { + javaInvocation1: null as LambdaInvocationDatadogData | null, + javaInvocation2: null as LambdaInvocationDatadogData | null, + dotnetInvocation1: null as LambdaInvocationDatadogData | null, + dotnetInvocation2: null as LambdaInvocationDatadogData | null, + pythonInvocation1: null as LambdaInvocationDatadogData | null, + pythonInvocation2: null as LambdaInvocationDatadogData | null, + }; + + beforeAll(async () => { + const javaBaseFunctionName = `integ-${identifier}-snapstart-java-lambda`; + const dotnetBaseFunctionName = `integ-${identifier}-snapstart-dotnet-lambda`; + const pythonBaseFunctionName = `integ-${identifier}-snapstart-python-lambda`; + + console.log('Publishing new versions for all Snapstart Lambda functions...'); + const javaVersion = await publishVersion(javaBaseFunctionName); + const dotnetVersion = await publishVersion(dotnetBaseFunctionName); + const pythonVersion = await publishVersion(pythonBaseFunctionName); + console.log(`All Lambda versions published successfully - Java: ${javaVersion}, .NET: ${dotnetVersion}, Python: ${pythonVersion}`); + + console.log('Waiting for SnapStart optimization to complete on all versions...'); + await waitForSnapStartReady(javaBaseFunctionName, javaVersion); + await waitForSnapStartReady(dotnetBaseFunctionName, dotnetVersion); + await waitForSnapStartReady(pythonBaseFunctionName, pythonVersion); + console.log('All SnapStart versions are ready'); + + console.log('Invoking all Snapstart Lambda functions concurrently...'); + const javaFunctionName = `${javaBaseFunctionName}:${javaVersion}`; + const dotnetFunctionName = `${dotnetBaseFunctionName}:${dotnetVersion}`; + const pythonFunctionName = `${pythonBaseFunctionName}:${pythonVersion}`; + const invocationResults = await Promise.all([ + invokeLambdaAndGetDatadogData(javaFunctionName, {}, false), + invokeLambdaAndGetDatadogData(javaFunctionName, {}, false), + invokeLambdaAndGetDatadogData(dotnetFunctionName, {}, false), + invokeLambdaAndGetDatadogData(dotnetFunctionName, {}, false), + invokeLambdaAndGetDatadogData(pythonFunctionName, {}, false), + invokeLambdaAndGetDatadogData(pythonFunctionName, {}, false), + ]); + + results.javaInvocation1 = invocationResults[0]; + results.javaInvocation2 = invocationResults[1]; + results.dotnetInvocation1 = invocationResults[2]; + results.dotnetInvocation2 = invocationResults[3]; + results.pythonInvocation1 = invocationResults[4]; + results.pythonInvocation2 = invocationResults[5]; + + console.log('All Snapstart Lambda invocations and data fetching completed'); + }, 900000); + + describe('Java Runtime with SnapStart', () => { + testSnapStartInvocation(() => results.javaInvocation1!); + testSnapStartInvocation(() => results.javaInvocation2!); + testTraceIsolation(() => results.javaInvocation1!, () => results.javaInvocation2!); + }); + + describe('.NET Runtime with SnapStart', () => { + testSnapStartInvocation(() => results.dotnetInvocation1!); + testSnapStartInvocation(() => results.dotnetInvocation2!); + testTraceIsolation(() => results.dotnetInvocation1!, () => results.dotnetInvocation2!); + }); + + // SVLS-5988 - Doesn't completely work as expected. + // describe('Python Runtime with SnapStart', () => { + // testSnapStartInvocation(() => results.pythonInvocation1!); + // testSnapStartInvocation(() => results.pythonInvocation2!); + // testTraceIsolation(() => results.pythonInvocation1!, () => results.pythonInvocation2!); + // }); + +}); + +function testSnapStartInvocation(getInvocation: () => LambdaInvocationDatadogData) { + it('should invoke successfully', () => { + const invocation = getInvocation(); + expect(invocation.statusCode).toBe(200); + }); + + it('should send one trace to Datadog', () => { + const invocation = getInvocation(); + expect(invocation.traces?.length).toEqual(1); + }); + + it('should have aws.lambda span with Snapstart properties', () => { + const invocation = getInvocation(); + const trace = invocation.traces![0]; + const awsLambdaSpan = trace.spans.find((span: any) => + span.attributes.operation_name === 'aws.lambda' + ); + expect(awsLambdaSpan).toBeDefined(); + expect(awsLambdaSpan?.attributes.custom.init_type).toBe('snap-start'); + }); + + it('should have aws.lambda.snapstart_restore span', () => { + const invocation = getInvocation(); + const trace = invocation.traces![0]; + const restoreSpan = trace.spans.find((span: any) => + span.attributes.operation_name === 'aws.lambda.snapstart_restore' + ); + expect(restoreSpan).toBeDefined(); + expect(restoreSpan).toMatchObject({ + attributes: { + operation_name: 'aws.lambda.snapstart_restore', + } + }); + + }); + + it('should NOT have aws.lambda.cold_start span (replaced by restore span)', () => { + const invocation = getInvocation(); + const trace = invocation.traces![0]; + const coldStartSpan = trace.spans.find((span: any) => + span.attributes.operation_name === 'aws.lambda.cold_start' + ); + expect(coldStartSpan).toBeUndefined(); + }); +} + +function testTraceIsolation(getInvocation1: () => LambdaInvocationDatadogData, getInvocation2: () => LambdaInvocationDatadogData) { + describe('Trace Isolation Between Concurrent Invocations', () => { + it('should have different trace IDs for each invocation', () => { + const invocation1 = getInvocation1(); + const invocation2 = getInvocation2(); + const trace1 = invocation1.traces![0]; + const trace2 = invocation2.traces![0]; + + expect(trace1.trace_id).not.toEqual(trace2.trace_id); + }); + }); +} + diff --git a/integration-tests/tests/utils/lambda.ts b/integration-tests/tests/utils/lambda.ts index e95aed23b..369b2cfdf 100644 --- a/integration-tests/tests/utils/lambda.ts +++ b/integration-tests/tests/utils/lambda.ts @@ -1,4 +1,4 @@ -import { LambdaClient, InvokeCommand, UpdateFunctionConfigurationCommand, GetFunctionConfigurationCommand } from '@aws-sdk/client-lambda'; +import { LambdaClient, InvokeCommand, UpdateFunctionConfigurationCommand, GetFunctionConfigurationCommand, PublishVersionCommand } from '@aws-sdk/client-lambda'; const lambdaClient = new LambdaClient({ region: 'us-east-1' }); @@ -57,24 +57,80 @@ export async function invokeLambda( } export async function forceColdStart(functionName: string): Promise { - const getConfigCommand = new GetFunctionConfigurationCommand({ - FunctionName: functionName, - }); - const config = await lambdaClient.send(getConfigCommand); + setTimestampEnvVar(functionName) + await new Promise(resolve => setTimeout(resolve, 10000)); +} - const updateCommand = new UpdateFunctionConfigurationCommand({ - FunctionName: functionName, - Environment: { - Variables: { - ...config.Environment?.Variables, - TS: Date.now().toString(), - }, - }, - }); - await lambdaClient.send(updateCommand); +export async function publishVersion(functionName: string): Promise { + console.debug(`Publishing version for ${functionName}`) + try { + await setTimestampEnvVar(functionName); + await new Promise(resolve => setTimeout(resolve, 10000)); + const command = new PublishVersionCommand({ + FunctionName: functionName, + }); + const response = await lambdaClient.send(command); + const version = response.Version || '$LATEST'; + console.debug(`Published version: ${version} for ${functionName}`); + return version; + } catch (error: any) { + console.error('Failed to publish Lambda version:', error.message); + throw error; + } +} - console.log(`Waiting 10 seconds for Lambda function ${functionName} to reinitialize...`); - await new Promise(resolve => setTimeout(resolve, 10000)); +export async function waitForSnapStartReady(functionName: string, version: string, timeoutMs: number = 300000): Promise { + const startTime = Date.now(); + while (Date.now() - startTime < timeoutMs) { + try { + const command = new GetFunctionConfigurationCommand({ + FunctionName: functionName, + Qualifier: version, + }); + const config = await lambdaClient.send(command); + + const optimizationStatus = config.SnapStart?.OptimizationStatus; + const state = config.State; + const lastUpdateStatus = config.LastUpdateStatus; + + if (optimizationStatus === 'On' && lastUpdateStatus === 'Successful') { + console.log(`SnapStart ready for ${functionName}:${version}`); + return; + } + + await new Promise(resolve => setTimeout(resolve, 10000)); + } catch (error: any) { + console.error(`Error checking SnapStart status: ${error.message}`); + await new Promise(resolve => setTimeout(resolve, 10_000)); + } + } + + throw new Error(`Timeout waiting for SnapStart optimization on ${functionName}:${version}`); } +export async function setTimestampEnvVar(functionName: string): Promise { + try { + const getConfigCommand = new GetFunctionConfigurationCommand({ + FunctionName: functionName, + }); + const currentConfig = await lambdaClient.send(getConfigCommand); + + const timestamp = Date.now().toString(); + const updatedEnvironment = { + Variables: { + ...(currentConfig.Environment?.Variables || {}), + ts: timestamp, + }, + }; + + const updateConfigCommand = new UpdateFunctionConfigurationCommand({ + FunctionName: functionName, + Environment: updatedEnvironment, + }); + await lambdaClient.send(updateConfigCommand); + } catch (error: any) { + console.error('Failed to set timestamp environment variable:', error.message); + throw error; + } +} \ No newline at end of file diff --git a/integration-tests/tests/utils/util.ts b/integration-tests/tests/utils/util.ts index 9cf45ef74..ac54ea331 100644 --- a/integration-tests/tests/utils/util.ts +++ b/integration-tests/tests/utils/util.ts @@ -15,8 +15,11 @@ export async function invokeLambdaAndGetDatadogData(functionName: string, payloa console.log('Waiting for logs and traces to be indexed in Datadog...'); await new Promise(resolve => setTimeout(resolve, 300000)); - const traces = await getTraces(functionName, result.requestId); - const logs = await getLogs(functionName, result.requestId); + // Strip alias suffix (e.g., ":snapstart") for Datadog queries since service name doesn't include it + const baseFunctionName = functionName.split(':')[0]; + + const traces = await getTraces(baseFunctionName, result.requestId); + const logs = await getLogs(baseFunctionName, result.requestId); const lambdaInvocationData: LambdaInvocationDatadogData = { requestId: result.requestId,