From 8c5db5634ccb91519b1a6638970883709f213ba5 Mon Sep 17 00:00:00 2001 From: Milan Garnier Date: Thu, 30 Jul 2026 11:44:55 +0200 Subject: [PATCH] fix(ci): wire GitLab CI to publish the system-tests image Uploads per-arch build artifacts to S3, gates on an open GitHub PR before uploading anything, then dispatches the GH Actions workflow that builds and pushes the multi-arch image. Gate now runs before the S3 upload so branches without an open PR never publish artifacts. Stacked on the workflow file itself (must land on master first, since workflow_dispatch resolves the workflow by filename against the default branch's registered workflows). --- .../gitlab-ci-publish-packages.sts.yaml | 2 +- .gitlab/generate-package.php | 150 ++++++++---------- 2 files changed, 71 insertions(+), 81 deletions(-) diff --git a/.github/chainguard/gitlab-ci-publish-packages.sts.yaml b/.github/chainguard/gitlab-ci-publish-packages.sts.yaml index ede956febc..4f9c1efd7f 100644 --- a/.github/chainguard/gitlab-ci-publish-packages.sts.yaml +++ b/.github/chainguard/gitlab-ci-publish-packages.sts.yaml @@ -3,5 +3,5 @@ issuer: https://gitlab.ddbuild.io subject_pattern: "project_path:DataDog/apm-reliability/dd-trace-php:ref_type:(branch|tag):ref:.*" permissions: - packages: write + actions: write pull_requests: read diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index f43076792a..3a45aeb694 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1558,8 +1558,10 @@ function appsec_image_from_tag_mapping(string $tag): string paths: - packages/datadog-setup.php -# Runs on every non-default branch so system tests can be run against any (non-default) in-progress branch. -"publish docker image for system tests (token)": +# System-tests image publishing. Runs on every non-default branch so system tests can be +# run against any in-progress branch (DataDog/system-tests LIBRARY_TARGET_BRANCH). + +"check open pr for system tests": stage: release image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 tags: [ "arch:amd64" ] @@ -1570,29 +1572,44 @@ function appsec_image_from_tag_mapping(string $tag): string - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: never - when: always - allow_failure: true variables: GIT_STRATEGY: none - script: - - dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages > github_token_system_tests.txt + script: | + set -e + # octo-sts token: pull_requests:read to gate the S3 upload on an open PR. + GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages) + + # Only publish for branches with an open PR (system tests pull the image per-PR), + # checked before anything is uploaded so throwaway branches never publish artifacts. + HTTP=$(curl -s -o /tmp/pulls.json -w "%{http_code}" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/DataDog/dd-trace-php/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open") + if [ "${HTTP}" != "200" ]; then + echo "ERROR: failed to check for an open PR on branch ${CI_COMMIT_BRANCH} (HTTP ${HTTP}):"; cat /tmp/pulls.json; exit 1 + fi + PR_COUNT=$(jq 'length' /tmp/pulls.json) + if [ "${PR_COUNT}" -eq 0 ]; then + echo "No open PR for branch ${CI_COMMIT_BRANCH}; skipping system-tests image publish." + echo "HAS_OPEN_PR=false" > pr_check.env + else + echo "Found open PR(s) on branch ${CI_COMMIT_BRANCH}" + echo "HAS_OPEN_PR=true" > pr_check.env + fi artifacts: - paths: - - github_token_system_tests.txt - expire_in: 1 hour - when: on_success + reports: + dotenv: pr_check.env -"publish docker image for system tests": +"publish docker image for system tests (upload)": stage: release - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble - tags: [ "docker-in-docker:amd64" ] - resource_group: "publish-system-tests-image-${CI_COMMIT_REF_SLUG}" - interruptible: true + image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.17.32 + tags: [ "arch:amd64" ] rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: never + - if: $HAS_OPEN_PR != "true" + when: never - when: always needs: - - job: "publish docker image for system tests (token)" + - job: "check open pr for system tests" artifacts: true - job: "datadog-setup.php" artifacts: true @@ -1602,74 +1619,47 @@ function appsec_image_from_tag_mapping(string $tag): string artifacts: true variables: GIT_STRATEGY: none - allow_failure: true script: | set -e - IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${CI_COMMIT_REF_SLUG}" - VERSIONS_URL="https://api.github.com/orgs/DataDog/packages/container/dd-trace-php%2Fdd-library-php/versions" - GITHUB_TOKEN=$( /dev/null - - STATUS=$(curl -s -o /tmp/pulls.json -w "%{http_code}" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/DataDog/dd-trace-php/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open") - if [ "${STATUS}" != "200" ]; then - echo "ERROR: failed to check for an open PR on branch ${CI_COMMIT_BRANCH} (HTTP ${STATUS}):"; cat /tmp/pulls.json - exit 1 - fi - PR_COUNT=$(jq 'length' /tmp/pulls.json) - if [ "${PR_COUNT}" -eq 0 ]; then - echo "No open PR for branch ${CI_COMMIT_BRANCH}; skipping system-tests image publish." - exit 3 - fi - echo "Found open PR(s) on branch ${CI_COMMIT_BRANCH}, continuing" - - docker login ghcr.io -u DataDog --password-stdin < github_token_system_tests.txt - echo "Docker login to ghcr.io succeeded" - - gh_api() { curl -s -o /tmp/resp.json -w "%{http_code}" -X "$1" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" "$2"; } - - # delete any pre-existing image with the same tag so pushing with the same tag doesn't pile up untagged versions. - echo "== Checking token and existing image tagged ${CI_COMMIT_REF_SLUG} ==" - STATUS=$(gh_api GET "$VERSIONS_URL") - if [ "$STATUS" != "200" ]; then - echo "WARNING: token check failed (HTTP ${STATUS}), skipping pre-delete:"; cat /tmp/resp.json - else - echo "Token OK (HTTP 200): listed package versions for dd-trace-php/dd-library-php" - VERSION_ID=$(jq -r --arg TAG "${CI_COMMIT_REF_SLUG}" \ - '.[] | select(.metadata.container.tags[]? == $TAG) | .id' /tmp/resp.json | head -n1) - if [ -z "$VERSION_ID" ]; then - echo "No pre-existing image tagged ${CI_COMMIT_REF_SLUG} found, nothing to delete" - else - STATUS=$(gh_api DELETE "${VERSIONS_URL}/${VERSION_ID}") - if [ "$STATUS" = "204" ]; then - echo "Deleted existing image version ${VERSION_ID} (HTTP 204)" - else - echo "WARNING: failed to delete existing image version ${VERSION_ID} (HTTP ${STATUS}), continuing anyway:" - cat /tmp/resp.json - fi - fi + # Normalise the arch tarball names so the GitHub Actions side needs only the branch slug. + DEST="s3://dd-trace-php-builds/ci/${CI_COMMIT_REF_SLUG}" + aws s3 cp packages/dd-library-php-*-x86_64-linux-gnu.tar.gz "${DEST}/dd-library-php-x86_64-linux-gnu.tar.gz" + aws s3 cp packages/dd-library-php-*-aarch64-linux-gnu.tar.gz "${DEST}/dd-library-php-aarch64-linux-gnu.tar.gz" + aws s3 cp packages/datadog-setup.php "${DEST}/datadog-setup.php" + echo "Uploaded system-tests artifacts to ${DEST}/" + +"publish docker image for system tests (dispatch)": + stage: release + image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 + tags: [ "arch:amd64" ] + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: never + - when: always + needs: + - job: "publish docker image for system tests (upload)" + variables: + GIT_STRATEGY: none + script: | + set -e + # octo-sts token: actions:write to dispatch the workflow. + GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages) + + # GHCR write must come from GitHub Actions' GITHUB_TOKEN, so trigger the workflow that + # pulls the artifacts back down from S3 and pushes the image. + PAYLOAD=$(jq -n --arg ref "${CI_DEFAULT_BRANCH}" --arg slug "${CI_COMMIT_REF_SLUG}" \ + '{ref: $ref, inputs: {ref_slug: $slug}}') + HTTP=$(curl -s -o /tmp/dispatch.json -w "%{http_code}" -X POST \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/DataDog/dd-trace-php/actions/workflows/publish-system-tests-image.yml/dispatches" \ + -d "${PAYLOAD}") + if [ "${HTTP}" != "204" ]; then + echo "ERROR: workflow_dispatch failed (HTTP ${HTTP}):"; cat /tmp/dispatch.json; exit 1 fi - - echo "== Building and pushing ${IMAGE} (linux/amd64, linux/arm64) ==" - # stage each arch's tarball under its own dir so the Dockerfile can pick the right one via buildx's TARGETARCH arg. - mkdir -p packages/amd64 packages/arm64 - cp packages/dd-library-php-*-x86_64-linux-gnu.tar.gz packages/amd64/ - cp packages/dd-library-php-*-aarch64-linux-gnu.tar.gz packages/arm64/ - cp packages/datadog-setup.php packages/amd64/ - cp packages/datadog-setup.php packages/arm64/ - - printf 'FROM scratch\nARG TARGETARCH\nCOPY packages/${TARGETARCH}/dd-library-php-*-linux-gnu.tar.gz /\nCOPY packages/${TARGETARCH}/datadog-setup.php /\n' \ - > Dockerfile.system-tests - BUILDER="system-tests-builder-${CI_JOB_ID}" - docker buildx create --name "$BUILDER" --driver docker-container - docker buildx build --builder "$BUILDER" --platform linux/amd64,linux/arm64 \ - -f Dockerfile.system-tests -t "$IMAGE" --push . - echo "Pushed $IMAGE" - after_script: - - rm -f github_token_system_tests.txt - - docker buildx rm "system-tests-builder-${CI_JOB_ID}" || true + echo "Dispatched publish-system-tests-image.yml (tag ${CI_COMMIT_REF_SLUG}) on ${CI_DEFAULT_BRANCH}" "bundle for reliability env": stage: shared-pipeline