Fix E2E test trigger (#1214)

## Overview
Change how the E2E tests are triggered. Launches pipelines for `amd64`
and `amd64, fips` flavors, will add tests for `arm64` and `arm64, fips`
as well if these work as expected.

## Testing 
E2E tests pass on this PR

Author: shreyamalpani

.gitlab/scripts/poll_e2e.sh

@@ -1,86 +1,52 @@
-#!/usr/bin/env bash
-# Trigger serverless-e2e-tests pipeline and poll until completion.
-
-set -euo pipefail
-
-if [ -z "${EXTENSION_LAYER_ARN:-}" ]; then
-    echo "ERROR: EXTENSION_LAYER_ARN is not set or empty"
-    exit 1
-fi
-
-E2E_PROJECT_ENCODED="DataDog%2Fserverless-e2e-tests"
-E2E_REF="${E2E_REF:-main}"
-
-curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv authanywhere-linux-amd64 /bin/authanywhere && chmod +x /bin/authanywhere
+#!/bin/bash
+curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" \
+    && mv "authanywhere-linux-amd64" /bin/authanywhere \
+    && chmod +x /bin/authanywhere
 
 BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci)
 
 BTI_RESPONSE=$(curl --silent --request GET \
     --header "$BTI_CI_API_TOKEN" \
     --header "Content-Type: application/vnd.api+json" \
-    "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=serverless-e2e-tests")
+    "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-extension")
 
 GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty')
+
 if [ -z "$GITLAB_TOKEN" ]; then
     echo "ERROR: could not obtain GitLab token from BTI"
     exit 1
 fi
 
-echo "Triggering DataDog/serverless-e2e-tests pipeline (ref: ${E2E_REF})..."
-echo "  EXTENSION_LAYER_ARN=${EXTENSION_LAYER_ARN}"
-
-TRIGGER_RESPONSE=$(curl --silent --request POST \
-    --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \
-    --header "Content-Type: application/json" \
-    --data "$(jq -n \
-        --arg ref "$E2E_REF" \
-        --arg arn "$EXTENSION_LAYER_ARN" \
-        '{ref: $ref, variables: [{key: "EXTENSION_VERSION", value: $arn}]}')" \
-    "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipeline")
+URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
 
-PIPELINE_ID=$(echo "$TRIGGER_RESPONSE" | jq -r '.id // empty')
-PIPELINE_URL=$(echo "$TRIGGER_RESPONSE" | jq -r '.web_url // empty')
-
-if [ -z "$PIPELINE_ID" ] || [ "$PIPELINE_ID" = "null" ]; then
-    echo "ERROR: failed to trigger downstream pipeline"
-    echo "Response: $TRIGGER_RESPONSE"
-    exit 1
-fi
-
-echo "Triggered downstream pipeline: ${PIPELINE_URL} (ID: ${PIPELINE_ID})"
+echo "Fetching E2E job status from: $URL"
 
 while true; do
-    STATUS=$(curl --silent \
-        --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" \
-        "${CI_API_V4_URL}/projects/${E2E_PROJECT_ENCODED}/pipelines/${PIPELINE_ID}" \
-        | jq -r '.status // empty')
-
-    echo -n "E2E pipeline ${PIPELINE_ID} status: ${STATUS}, "
-
-    case "$STATUS" in
-        success)
-            echo "E2E tests passed"
-            exit 0
-            ;;
-        failed)
-            echo "E2E tests failed"
-            exit 1
-            ;;
-        canceled|canceling)
-            echo "E2E tests canceled"
-            exit 1
-            ;;
-        skipped)
-            echo "E2E tests skipped"
-            exit 0
-            ;;
-        running|pending|created|waiting_for_resource|preparing)
-            echo "still running, checking again in 2 minutes..."
-            ;;
-        *)
-            echo "unknown status, checking again in 2 minutes..."
-            ;;
-    esac
-
+    HTTP_STATUS=$(curl -s -o /tmp/e2e_response.json -w "%{http_code}" --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL")
+    if [ "$HTTP_STATUS" != "200" ]; then
+        echo "WARNING: GitLab API returned HTTP $HTTP_STATUS, retrying in 2 minutes..."
+        sleep 120
+        continue
+    fi
+    RESPONSE=$(cat /tmp/e2e_response.json)
+    E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r --arg name "$E2E_JOB_NAME" '.[] | select(.name==$name) | .downstream_pipeline.status')
+    echo -n "E2E job status: $E2E_JOB_STATUS, "
+    if [ "$E2E_JOB_STATUS" == "success" ]; then
+        echo "E2E tests completed successfully"
+        exit 0
+    elif [ "$E2E_JOB_STATUS" == "failed" ]; then
+        echo "E2E tests failed"
+        exit 1
+    elif [ "$E2E_JOB_STATUS" == "running" ]; then
+        echo "E2E tests are still running, retrying in 2 minutes..."
+    elif [ "$E2E_JOB_STATUS" == "canceled" ]; then
+        echo "E2E tests were canceled"
+        exit 1
+    elif [ "$E2E_JOB_STATUS" == "skipped" ]; then
+        echo "E2E tests were skipped"
+        exit 0
+    else
+        echo "Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..."
+    fi
     sleep 120
 done

.gitlab/templates/pipeline.yaml.tpl

@@ -231,7 +231,7 @@ publish layer [self-monitoring] ({{ $flavor.name }}):
 {{ end }}  # end flavors
 
 {{ range $f := (ds "flavors").flavors }}
-{{ if $f.needs_layer_publish }}
+{{ if and $f.needs_layer_publish (eq $f.arch "amd64") }}
 {{- $dotenvE2E := printf "%s_sandbox_e2e.env" $f.suffix }}
 {{ with $environment := (ds "environments").environments.sandbox }}
 
@@ -270,20 +270,33 @@ publish layer e2e sandbox ({{ $f.name }}):
   script:
     - .gitlab/scripts/publish_layers.sh
 
-e2e-suite ({{ $f.name }}):
+e2e-test ({{ $f.name }}):
   stage: e2e
-  image: registry.ddbuild.io/images/docker:20.10-py3
-  tags: ["arch:amd64"]
-  timeout: 3h
+  trigger:
+    project: DataDog/serverless-e2e-tests
+    strategy: depend
   rules:
-    # Do not block on release, in case E2E is flaky
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
       when: on_success
       allow_failure: true
     - when: on_success
   needs:
     - job: "publish layer e2e sandbox ({{ $f.name }})"
       artifacts: true
+  variables:
+    EXTENSION_VERSION: $EXTENSION_LAYER_ARN
+    ARCHITECTURE: {{ $f.arch }}
+
+e2e-test-status ({{ $f.name }}):
+  stage: e2e
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  tags: ["arch:amd64"]
+  timeout: 3h
+  variables:
+    E2E_JOB_NAME: "e2e-test ({{ $f.name }})"
+  needs:
+    - job: "publish layer e2e sandbox ({{ $f.name }})"
+      artifacts: false
   script:
     - .gitlab/scripts/poll_e2e.sh