From af860efdd00b6dc0af3d481d35b4aad5f34ba1c8 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Mon, 31 Mar 2025 15:00:42 +0200 Subject: [PATCH 01/10] [AWSX-1559] feat: update Python version to 3.13 --- .github/workflows/aws_integration_test.yml | 2 +- .github/workflows/aws_lint.yml | 2 +- .github/workflows/aws_unit_test.yml | 4 ++-- aws/logs_monitoring/setup.py | 4 ++-- aws/logs_monitoring/template.yaml | 4 ++-- aws/logs_monitoring/tools/build_bundle.sh | 2 +- .../tools/integration_tests/integration_tests.sh | 8 ++++---- .../trace_forwarder/scripts/build_linux_go_bin.sh | 2 +- aws/logs_monitoring/trace_forwarder/scripts/run_tests.sh | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/aws_integration_test.yml b/.github/workflows/aws_integration_test.yml index c4b375ccc..451fddf2f 100644 --- a/.github/workflows/aws_integration_test.yml +++ b/.github/workflows/aws_integration_test.yml @@ -28,4 +28,4 @@ jobs: - name: AWS Python integration tests run: | - ./aws/logs_monitoring/tools/integration_tests/integration_tests.sh --python-version=3.12 + ./aws/logs_monitoring/tools/integration_tests/integration_tests.sh --python-version=3.13 diff --git a/.github/workflows/aws_lint.yml b/.github/workflows/aws_lint.yml index 4ec1ce5f6..907102690 100644 --- a/.github/workflows/aws_lint.yml +++ b/.github/workflows/aws_lint.yml @@ -11,7 +11,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 with: - python-version: "3.12" + python-version: "3.13" - name: Install pip run: | diff --git a/.github/workflows/aws_unit_test.yml b/.github/workflows/aws_unit_test.yml index ef406d7f1..2f9556a75 100644 --- a/.github/workflows/aws_unit_test.yml +++ b/.github/workflows/aws_unit_test.yml @@ -8,10 +8,10 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v3 - - name: Setup Python 3.12 + - name: Setup Python 3.13 uses: actions/setup-python@v3 with: - python-version: 3.12 + python-version: 3.13 - name: AWS Python unit tests env: diff --git a/aws/logs_monitoring/setup.py b/aws/logs_monitoring/setup.py index 2fc4a9a5f..b61ec3193 100644 --- a/aws/logs_monitoring/setup.py +++ b/aws/logs_monitoring/setup.py @@ -12,10 +12,10 @@ author="Datadog, Inc.", author_email="dev@datadoghq.com", classifiers=[ - "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], keywords="datadog aws lambda layer", - python_requires=">=3.12, <3.13", + python_requires=">=3.13, <3.14", extras_require={ "dev": ["nose2==0.9.1", "flake8==3.7.9", "requests==2.22.0", "boto3==1.10.33"] }, diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml index b7d0b14fa..9fc24c77b 100644 --- a/aws/logs_monitoring/template.yaml +++ b/aws/logs_monitoring/template.yaml @@ -432,7 +432,7 @@ Resources: - {DdForwarderVersion: !FindInMap [Constants, DdForwarder, Version]} - ZipFile: " " MemorySize: !Ref MemorySize - Runtime: python3.12 + Runtime: python3.13 Architectures: - arm64 Timeout: !Ref Timeout @@ -843,7 +843,7 @@ Resources: Properties: Description: Copies Datadog Forwarder zip to the destination S3 bucket Handler: index.handler - Runtime: python3.12 + Runtime: python3.13 Timeout: 600 Code: ZipFile: | diff --git a/aws/logs_monitoring/tools/build_bundle.sh b/aws/logs_monitoring/tools/build_bundle.sh index b379dbfe4..1f0669c4a 100755 --- a/aws/logs_monitoring/tools/build_bundle.sh +++ b/aws/logs_monitoring/tools/build_bundle.sh @@ -42,7 +42,7 @@ else VERSION=$1 fi -PYTHON_VERSION="${PYTHON_VERSION:-3.12}" +PYTHON_VERSION="${PYTHON_VERSION:-3.13}" FORWARDER_PREFIX="aws-dd-forwarder" FORWARDER_DIR="../.forwarder" diff --git a/aws/logs_monitoring/tools/integration_tests/integration_tests.sh b/aws/logs_monitoring/tools/integration_tests/integration_tests.sh index fd8db6084..70e884212 100755 --- a/aws/logs_monitoring/tools/integration_tests/integration_tests.sh +++ b/aws/logs_monitoring/tools/integration_tests/integration_tests.sh @@ -7,9 +7,9 @@ set -e -PYTHON_VERSION="python3.12" -PYTHON_VERSION_TAG="3.12" -PYTHON_IMAGE="python:3.12" +PYTHON_VERSION="python3.13" +PYTHON_VERSION_TAG="3.13" +PYTHON_IMAGE="python:3.13" SKIP_FORWARDER_BUILD=false UPDATE_SNAPSHOTS=false LOG_LEVEL=info @@ -36,7 +36,7 @@ for arg in "$@"; do SKIP_FORWARDER_BUILD=true shift ;; - + # -u or --update # Update the snapshots to reflect this test run diff --git a/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh b/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh index 5998bf949..aa15db539 100755 --- a/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh +++ b/aws/logs_monitoring/trace_forwarder/scripts/build_linux_go_bin.sh @@ -17,7 +17,7 @@ rm -rf ./bin # between different python runtimes. if [[ $(docker image ls | grep -c golang) -lt 1 ]]; then - docker buildx build --platform linux/arm64 -t golang . --no-cache --build-arg "runtime=python:3.12" + docker buildx build --platform linux/arm64 -t golang . --no-cache --build-arg "runtime=python:3.13" fi id=$(docker create --platform linux/arm64 golang) diff --git a/aws/logs_monitoring/trace_forwarder/scripts/run_tests.sh b/aws/logs_monitoring/trace_forwarder/scripts/run_tests.sh index 23ec52b22..dabc3cbc1 100755 --- a/aws/logs_monitoring/trace_forwarder/scripts/run_tests.sh +++ b/aws/logs_monitoring/trace_forwarder/scripts/run_tests.sh @@ -11,5 +11,5 @@ set -e # Change to the parent of the directory this script is in cd $(dirname "$0")/.. -docker buildx build --platform linux/arm64 -t datadog-go-layer . --build-arg runtime=python:3.12 +docker buildx build --platform linux/arm64 -t datadog-go-layer . --build-arg runtime=python:3.13 docker run --rm datadog-go-layer go test -v ./... From 2666be4f0bb593d3d1fe8b950d343779b282fe54 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Mon, 31 Mar 2025 15:11:29 +0200 Subject: [PATCH 02/10] [AWSX-1559] feat: update version --- .../tools/integration_tests/cache_test_lambda/serverless.yml | 2 +- aws/logs_monitoring/tools/publish_layers.sh | 2 +- aws/vpc_flow_log_monitoring/vpc-flow-log-sam-template.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/logs_monitoring/tools/integration_tests/cache_test_lambda/serverless.yml b/aws/logs_monitoring/tools/integration_tests/cache_test_lambda/serverless.yml index 37a90a141..6bcab84fc 100644 --- a/aws/logs_monitoring/tools/integration_tests/cache_test_lambda/serverless.yml +++ b/aws/logs_monitoring/tools/integration_tests/cache_test_lambda/serverless.yml @@ -1,7 +1,7 @@ service: integration-tests provider: name: aws - runtime: python3.12 + runtime: python3.13 functions: cache_test_lambda: diff --git a/aws/logs_monitoring/tools/publish_layers.sh b/aws/logs_monitoring/tools/publish_layers.sh index 6dd93b59d..7be1488af 100755 --- a/aws/logs_monitoring/tools/publish_layers.sh +++ b/aws/logs_monitoring/tools/publish_layers.sh @@ -92,7 +92,7 @@ user_confirm() { # Makes sure any subprocesses will be terminated with this process trap "pkill -P $$; exit 1;" INT -PYTHON_VERSIONS_FOR_AWS_CLI=("python3.12") +PYTHON_VERSIONS_FOR_AWS_CLI=("python3.13") LAYER_PATHS=(".forwarder/aws-dd-forwarder-${FORWARDER_VERSION}-layer.zip") AVAILABLE_LAYERS=("Datadog-Forwarder") AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') diff --git a/aws/vpc_flow_log_monitoring/vpc-flow-log-sam-template.yaml b/aws/vpc_flow_log_monitoring/vpc-flow-log-sam-template.yaml index b084dbaab..023ac6846 100644 --- a/aws/vpc_flow_log_monitoring/vpc-flow-log-sam-template.yaml +++ b/aws/vpc_flow_log_monitoring/vpc-flow-log-sam-template.yaml @@ -18,7 +18,7 @@ Resources: Policies: KMSDecryptPolicy: KeyId: !Ref KMSKeyId - Runtime: python3.12 + Runtime: python3.13 Architectures: - arm64 Timeout: 10 From 85aef8c6f4ffb46b861ba29fe7f3491ff17f9ed2 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Tue, 1 Apr 2025 10:51:25 +0200 Subject: [PATCH 03/10] [AWSX-1559] chore: update version --- aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml b/aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml index 22443fbe1..cc318ea5a 100644 --- a/aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml +++ b/aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml @@ -20,7 +20,7 @@ Resources: KeyId: !Ref KMSKeyId Architectures: - arm64 - Runtime: python3.12 + Runtime: python3.13 Timeout: 10 KmsKeyArn: !Sub From 96fea8ecadc543ee7fb7b1c1679d3a44036304f9 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Tue, 8 Jul 2025 13:21:14 +0200 Subject: [PATCH 04/10] [AWSX-1559] feat: bump cfn-lint to 1.19.0 --- .github/workflows/aws_lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/aws_lint.yml b/.github/workflows/aws_lint.yml index 907102690..1c3077737 100644 --- a/.github/workflows/aws_lint.yml +++ b/.github/workflows/aws_lint.yml @@ -33,7 +33,7 @@ jobs: - name: Install cfn-lint print version and check formatting run: | - pip install cfn-lint==1.11.1 + pip install cfn-lint==1.19.0 cfn-lint --version cfn-lint -t aws/logs_monitoring/template.yaml cfn-lint -t aws/rds_enhanced_monitoring/rds-enhanced-sam-template.yaml From 115f37566349aec236af0bccfb706e7630e2fc35 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Tue, 8 Jul 2025 13:27:20 +0200 Subject: [PATCH 05/10] [AWSX-1559] feat: use ddtrace v3.0.0 --- aws/logs_monitoring/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/requirements.txt b/aws/logs_monitoring/requirements.txt index 4d729fdc4..ddaa2f898 100644 --- a/aws/logs_monitoring/requirements.txt +++ b/aws/logs_monitoring/requirements.txt @@ -6,7 +6,7 @@ charset-normalizer datadog-lambda==6.104.0 datadog==0.50.2 ddsketch==3.0.1 -ddtrace==2.17.3 +ddtrace>=3.0.0 deprecated envier exceptiongroup From fd621266981a23255f4f1b7a43b824d0dcabddff Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Tue, 8 Jul 2025 13:44:57 +0200 Subject: [PATCH 06/10] [AWSX-1559] chore: update python version in docs --- aws/logs_monitoring/README.md | 6 +++--- aws/vpc_flow_log_monitoring/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/logs_monitoring/README.md b/aws/logs_monitoring/README.md index 1a3e6c6bb..1398b39b8 100644 --- a/aws/logs_monitoring/README.md +++ b/aws/logs_monitoring/README.md @@ -115,7 +115,7 @@ resource "aws_cloudformation_stack" "datadog_forwarder" { If you can't install the Forwarder using the provided CloudFormation template, you can install the Forwarder manually following the steps below. Feel free to open an issue or pull request to let us know if there is anything we can improve to make the template work for you. -1. Create a Python 3.12 Lambda function using `aws-dd-forwarder-.zip` from the latest [releases][101]. +1. Create a Python 3.13 Lambda function using `aws-dd-forwarder-.zip` from the latest [releases][101]. 2. Save your [Datadog API key][102] in AWS Secrets Manager, set environment variable `DD_API_KEY_SECRET_ARN` with the secret ARN on the Lambda function, and add the `secretsmanager:GetSecretValue` permission to the Lambda execution role. 3. If you need to forward logs from S3 buckets, add the `s3:GetObject` permission to the Lambda execution role. 4. Set the environment variable `DD_ENHANCED_METRICS` to `false` on the Forwarder. This stops the Forwarder from generating enhanced metrics itself, but it will still forward custom metrics from other lambdas.DdFetchLambdaTags @@ -149,7 +149,7 @@ The environment variables provided on this If you encounter issues upgrading to the latest version, check the Troubleshooting section. ### Upgrade an older verison to 4.3.0+ -Starting verison 4.3.0 Lambda forwarder will support a single python version only. The supported Python version of this release is 3.12. +Starting verison 4.3.0 Lambda forwarder will support a single python version only. The supported Python version of this release is 3.13. ### Upgrade an older version to +4.0.0 Starting version 4.0.0 `source`, `service` and `host` identification logic will be pulled out from the Lambda forwarder's code and set in directly in Datadog's backend. The first migrated log source is `RDS`. @@ -748,7 +748,7 @@ The value of the `service` tag is determined based on multiple inputs. These inp 1. Log message custom tags: If the log message has a `ddtags` key which contains a `service` tag value, it will be used to override the `service` tag in the log event. 2. Lambda tags cache (applicable for Lambda logs only): Activating `DdFetchLambdaTags` will fetch and store all Lambda functions tags and will override the `service` tag if it wasn't set previously or was set to a default value i.e. `source` value. 3. Cloudwatch log group tags cache (applicable for Cloudwatch logs only): Activating `DdFetchLogGroupTags` will fetch and store all Cloudwatch log groups tags which are added to the `ddtags` entry in the log event. If `service` tag value was set in the tags cache it will be used to set the `service` tag for the log event. -4. Directly setting a `service` tag value in the forwarder's `ddtags` ENV var. +4. Directly setting a `service` tag value in the forwarder's `ddtags` ENV var. 5. Default value equal to the `source` tag. diff --git a/aws/vpc_flow_log_monitoring/README.md b/aws/vpc_flow_log_monitoring/README.md index 8140c0d26..7cb40db30 100644 --- a/aws/vpc_flow_log_monitoring/README.md +++ b/aws/vpc_flow_log_monitoring/README.md @@ -49,7 +49,7 @@ version, account, eni, source, destination, srcport, destport="22", protocol="6" - Create a `lambda_execution` role and attach this policy - - Create a lambda function: Skip the blueprint, name it `functionname`, set the Runtime to `Python 3.12`, the Architecture to `arm64`,, the handle to `lambda_function.lambda_handler`, and the role to `lambda_execution`. + - Create a lambda function: Skip the blueprint, name it `functionname`, set the Runtime to `Python 3.13`, the Architecture to `arm64`,, the handle to `lambda_function.lambda_handler`, and the role to `lambda_execution`. - Copy the content of `functionname/lambda_function.py` in the code section, make sure to update the `KMS_ENCRYPTED_KEYS` environment variable with the encrypted key generated in step 1 From 331ca543e407927e3fde0ea0c83efcb2f76972d7 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Tue, 8 Jul 2025 14:53:42 +0200 Subject: [PATCH 07/10] [AWSX-1559] chore: set ddtrace version to 3.0.0 instead of >= --- aws/logs_monitoring/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/requirements.txt b/aws/logs_monitoring/requirements.txt index ddaa2f898..d7c6ba4b6 100644 --- a/aws/logs_monitoring/requirements.txt +++ b/aws/logs_monitoring/requirements.txt @@ -6,7 +6,7 @@ charset-normalizer datadog-lambda==6.104.0 datadog==0.50.2 ddsketch==3.0.1 -ddtrace>=3.0.0 +ddtrace==3.0.0 deprecated envier exceptiongroup From 977947251c8e979724adc9c8764fab3aeb295eab Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Wed, 23 Jul 2025 11:30:26 +0200 Subject: [PATCH 08/10] [AWSX-1559] refactor: update requirements for ddd-lambda, dd, ddtrace --- aws/logs_monitoring/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/logs_monitoring/requirements.txt b/aws/logs_monitoring/requirements.txt index d7c6ba4b6..6c9cbe8f3 100644 --- a/aws/logs_monitoring/requirements.txt +++ b/aws/logs_monitoring/requirements.txt @@ -3,10 +3,10 @@ bytecode cattrs certifi charset-normalizer -datadog-lambda==6.104.0 -datadog==0.50.2 +datadog-lambda==7.112.0 +datadog==0.52.0 ddsketch==3.0.1 -ddtrace==3.0.0 +ddtrace==3.10.2 deprecated envier exceptiongroup From 365a4c6232c13ef97e78864167c874e7d05221bf Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Wed, 23 Jul 2025 11:38:24 +0200 Subject: [PATCH 09/10] [AWSX-1559] chore: update version in docs for next forwarder version --- aws/logs_monitoring/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws/logs_monitoring/README.md b/aws/logs_monitoring/README.md index 1398b39b8..51539a76e 100644 --- a/aws/logs_monitoring/README.md +++ b/aws/logs_monitoring/README.md @@ -148,8 +148,11 @@ The environment variables provided on this If you encounter issues upgrading to the latest version, check the Troubleshooting section. +### Upgrade an older verison to 4.12.0+ +The supported Python version of this release is 3.13. + ### Upgrade an older verison to 4.3.0+ -Starting verison 4.3.0 Lambda forwarder will support a single python version only. The supported Python version of this release is 3.13. +Starting verison 4.3.0 Lambda forwarder will support a single python version only. The supported Python version of this release is 3.12. ### Upgrade an older version to +4.0.0 Starting version 4.0.0 `source`, `service` and `host` identification logic will be pulled out from the Lambda forwarder's code and set in directly in Datadog's backend. The first migrated log source is `RDS`. From d1ae1f44eabaed0a453c427e28c06852db103587 Mon Sep 17 00:00:00 2001 From: Sanassi Mory Cisse Date: Wed, 23 Jul 2025 11:47:40 +0200 Subject: [PATCH 10/10] [AWSX-1559] refactor: apply doc suggestion --- aws/logs_monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/logs_monitoring/README.md b/aws/logs_monitoring/README.md index 51539a76e..50178a4e3 100644 --- a/aws/logs_monitoring/README.md +++ b/aws/logs_monitoring/README.md @@ -149,7 +149,7 @@ The environment variables provided on this If you encounter issues upgrading to the latest version, check the Troubleshooting section. ### Upgrade an older verison to 4.12.0+ -The supported Python version of this release is 3.13. +Starting version 4.12.0+ Lambda function has been updated to require **Python 3.13**. If upgrading an older forwarder installation to 4.12.0+, ensure AWS Lambda function is configured to use Python 3.13 ### Upgrade an older verison to 4.3.0+ Starting verison 4.3.0 Lambda forwarder will support a single python version only. The supported Python version of this release is 3.12.