Merge branch 'main' into joey/enhance-build-scripts

Author: joeyzhao2018

.github/workflows/build.yml

@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Set up Node 18.12
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1
         with:
           node-version: 18.12
 
@@ -24,7 +24,7 @@ jobs:
 
       - name: Restore node modules from cache
         id: cache-node-modules
-        uses: actions/cache@v3
+        uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
@@ -47,10 +47,10 @@ jobs:
         node-version: [18.12, 20.19, 22.11, 24.11]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Set up Node ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1
         with:
           node-version: ${{ matrix.node-version }}
 
@@ -60,7 +60,7 @@ jobs:
 
       - name: Restore node modules from cache
         id: cache-node-modules
-        uses: actions/cache@v3
+        uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}

.github/workflows/build_layer.yml

@@ -0,0 +1,36 @@
+name: Build Layers for System Tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - "main"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node_version: ["18.12", "20.19", "22.11", "24.11"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
+      - name: Build layer for Node ${{ matrix.node_version }}
+        run: |
+          NODE_MAJOR=$(echo "${{ matrix.node_version }}" | cut -d '.' -f 1)
+          temp_dir=$(mktemp -d)
+          docker buildx build -t datadog-lambda-layer-node:${{ matrix.node_version }} . --no-cache \
+            --build-arg image=node:${NODE_MAJOR}-bullseye --progress=plain -o $temp_dir/nodejs
+          mkdir -p .layers
+          (cd $temp_dir && zip -q -r $GITHUB_WORKSPACE/.layers/datadog_lambda_node${{ matrix.node_version }}.zip ./)
+          rm -rf $temp_dir
+
+      - name: Upload layer artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          path: .layers/datadog_lambda_node${{ matrix.node_version }}.zip
+          name: datadog_lambda_node${{ matrix.node_version }}

.github/workflows/codeql-analysis.yml

@@ -38,11 +38,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2.28.1
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2.28.1
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -67,4 +67,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2.28.1

.github/workflows/update-deps.yml

@@ -18,11 +18,11 @@ jobs:
           app-id: ${{ secrets.GH_APP_ID }}
           private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
         with:
           token: ${{ steps.generate_token.outputs.token }}
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1
         with:
           node-version: "20.x"
 
@@ -32,7 +32,7 @@ jobs:
           yarn upgrade
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@18f7dc018cc2cd597073088f7c7591b9d1c02672 # v3.14.0
         with:
           token: ${{ steps.generate_token.outputs.token }}
           commit-message: update dependencies

.github/workflows/update-snapshots.yml

@@ -8,16 +8,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
       - name: Set up Node 14
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@3235b876344d2a9aa001b8d1453c930bba69e610 # v3.9.1
         with:
           node-version: 14
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v2.10.0
 
       - name: Complete Buildx Setup
         run:  docker run --privileged --rm tonistiigi/binfmt --install all
@@ -28,7 +28,7 @@ jobs:
 
       - name: Restore node modules from cache
         id: cache-node-modules
-        uses: actions/cache@v3
+        uses: actions/cache@6f8efc29b200d32929f49075959781ed54ec270c # v3.5.0
         with:
           path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
@@ -52,7 +52,7 @@ jobs:
         run: ./scripts/run_integration_tests.sh
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@18f7dc018cc2cd597073088f7c7591b9d1c02672 # v3.14.0
         with:
           commit-message: update snapshots
           title: Update Snapshots

.gitlab-ci.yml

@@ -10,6 +10,9 @@ ci image:
   stage: build
   image: registry.ddbuild.io/images/docker:20.10
   tags: ["arch:arm64"]
+  id_tokens:
+    DDSIGN_ID_TOKEN:
+      aud: image-integrity
   needs: []
   rules:
     - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'
@@ -19,7 +22,9 @@ ci image:
   variables:
     DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION}
   script:
-    - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile .
+    - METADATA_FILE=$(mktemp)
+    - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull ---tag ${DOCKER_TARGET} -f .gitlab/Dockerfile -push --metadata-file ${METADATA_FILE} .
+    - ddsign sign ${DOCKER_TARGET} --docker-metadata-file ${METADATA_FILE}
 
 .go-cache: &go-cache
   key: datadog-lambda-js-go-cache

.gitlab/input_files/build.yaml.tpl

@@ -1,3 +1,5 @@
+{{- $e2e_region := "us-west-2" -}}
+
 variables:
   CI_DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-js
   CI_DOCKER_TARGET_VERSION: latest
@@ -7,6 +9,7 @@ stages:
  - test
  - sign
  - publish
+ - e2e
 
 default:
   retry:
@@ -94,6 +97,7 @@ integration test ({{ $runtime.name }}):
     - RUNTIME_PARAM={{ $runtime.node_major_version }} ./scripts/run_integration_tests.sh
 
 {{ range $environment := (ds "environments").environments }}
+{{ $dotenv := print $runtime.name "_" $environment.name ".env" }}
 
 {{ if or (eq $environment.name "prod") }}
 sign layer ({{ $runtime.name }}):
@@ -126,10 +130,15 @@ publish layer {{ $environment.name }} ({{ $runtime.name }}):
   tags: ["arch:amd64"]
   image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
   rules:
+    - if: '"{{ $environment.name }}" == "sandbox" && $REGION == "{{ $e2e_region }}"'
+      when: on_success
     - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/'
       when: manual
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
+  artifacts:
+    reports:
+      dotenv: {{ $dotenv }}
   needs:
 {{ if or (eq $environment.name "prod") }}
       - sign layer ({{ $runtime.name }})
@@ -154,7 +163,7 @@ publish layer {{ $environment.name }} ({{ $runtime.name }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
   script:
-    - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} .gitlab/scripts/publish_layers.sh
+    - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} DOTENV={{ $dotenv }} .gitlab/scripts/publish_layers.sh
 
 {{- end }}
 
@@ -203,3 +212,27 @@ publish npm package:
     - mkdir -p datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID}
     - cp .layers/datadog_lambda_node*.zip datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID}
 {{ end }}
+
+e2e-test:
+  stage: e2e
+  trigger:
+    project: DataDog/serverless-e2e-tests
+    strategy: depend
+  variables:
+    LANGUAGES_SUBSET: node
+    {{- range (ds "runtimes").runtimes }}
+    {{- $version := print (.name | strings.Trim "node") }}
+    NODEJS_{{ $version }}_VERSION: $NODE_{{ $version }}_VERSION
+    {{- end }}
+  needs: {{ range (ds "runtimes").runtimes }}
+    - "publish layer sandbox ({{ .name }}): [{{ $e2e_region }}]"
+    {{- end }}
+
+
+e2e-test-status:
+  stage: e2e
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  tags: ["arch:amd64"]
+  timeout: 3h
+  script:
+      - .gitlab/scripts/poll_e2e.sh

.gitlab/scripts/poll_e2e.sh

@@ -0,0 +1,38 @@
+curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv "authanywhere-linux-amd64" /bin/authanywhere && chmod +x /bin/authanywhere
+
+BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci)
+
+BTI_RESPONSE=$(curl --silent --request GET \
+    --header "$BTI_CI_API_TOKEN" \
+    --header "Content-Type: application/vnd.api+json" \
+    "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-js")
+
+GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') 
+
+URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
+
+echo "Fetching E2E job status from: $URL"
+
+while true; do
+    RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL")
+    E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .downstream_pipeline.status')
+    echo -n "E2E job status: $E2E_JOB_STATUS, "
+    if [ "$E2E_JOB_STATUS" == "success" ]; then
+        echo "✅ E2E tests completed successfully"
+        exit 0
+    elif [ "$E2E_JOB_STATUS" == "failed" ]; then
+        echo "❌ E2E tests failed"
+        exit 1
+    elif [ "$E2E_JOB_STATUS" == "running" ]; then
+        echo "⏳ E2E tests are still running, retrying in 2 minutes..."
+    elif [ "$E2E_JOB_STATUS" == "canceled" ]; then
+        echo "🚫 E2E tests were canceled"
+        exit 1
+    elif [ "$E2E_JOB_STATUS" == "skipped" ]; then
+        echo "⏭️ E2E tests were skipped"
+        exit 0
+    else
+        echo "❓ Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..."
+    fi
+    sleep 120
+done

.gitlab/scripts/publish_layers.sh

@@ -146,6 +146,8 @@ fi
 while [ $latest_version -lt $VERSION ]; do
     latest_version=$(publish_layer $REGION $layer $aws_cli_node_version_key $layer_path)
     printf "[$REGION] Published version $latest_version for layer $layer in region $REGION\n"
+    latest_arn=$(aws lambda get-layer-version --layer-name $layer --version-number $latest_version --region $REGION --query 'LayerVersionArn' --output text)
+    printf "[$REGION] Published arn $latest_arn\n"
 
     # This shouldn't happen unless someone manually deleted the latest version, say 28, and
     # then tries to republish 28 again. The published version would actually be 29, because
@@ -156,4 +158,12 @@ while [ $latest_version -lt $VERSION ]; do
     fi
 done
 
+if [ -n "$DOTENV" ]; then
+    printf "[$REGION] Exporting layer version to $DOTENV file...\n"
+    node_version=$NODE_VERSION
+    major_version="${node_version:0:2}"
+    echo "NODE_${major_version}_VERSION=$latest_arn" >> "$DOTENV"
+    cat "$DOTENV"
+fi
+
 printf "[$REGION] Finished publishing layers...\n\n"