fix(ci): publish system-tests image via S3 + GH Actions dispatch

#4013 pushed the per-branch system-tests image to GHCR directly from
GitLab CI, authenticating with an octo-sts token scoped
`packages: write`. That token comes from a GitHub App installation,
and GHCR refuses package writes from installation tokens
(permission_denied: installation not allowed to Write organization
package). The job has allow_failure: true, so it has been failing
silently since #4013 merged and no image has actually been published.

Replace it with a two-step, two-runtime pipeline:

1. GitLab CI ("publish docker image for system tests (upload)")
   uploads the built tarballs and installer to
   s3://dd-trace-php-builds/ci/<ref-slug>/, then
   ("... (dispatch)") mints an actions: write-scoped octo-sts token
   and dispatches a new GitHub Actions workflow.
2. .github/workflows/publish-system-tests-image.yml downloads the
   artifacts from S3 and pushes to GHCR using the workflow's native
   GITHUB_TOKEN, which is actually allowed to write org packages.

Update gitlab-ci-publish-packages.sts.yaml from packages: write to
actions: write to match, since nothing mints a packages-scoped token
through this policy anymore.

Author: MilanGarnier

.github/chainguard/gitlab-ci-publish-packages.sts.yaml

@@ -3,5 +3,5 @@ issuer: https://gitlab.ddbuild.io
 subject_pattern: "project_path:DataDog/apm-reliability/dd-trace-php:ref_type:(branch|tag):ref:.*"
 
 permissions:
-  packages: write
+  actions: write
   pull_requests: read

.github/workflows/publish-system-tests-image.yml

@@ -0,0 +1,68 @@
+name: "Publish system-tests image"
+
+# Pushes the per-branch dd-library-php image that DataDog/system-tests pulls
+# (LIBRARY_TARGET_BRANCH), tagged with the GitLab CI_COMMIT_REF_SLUG.
+on:
+  workflow_dispatch:
+    inputs:
+      ref_slug:
+        description: "Sanitised branch name (GitLab CI_COMMIT_REF_SLUG); used as the image tag and the S3 prefix."
+        required: true
+
+concurrency:
+  group: "publish-system-tests-image-${{ github.event.inputs.ref_slug }}"
+  cancel-in-progress: true
+
+env:
+  IMAGE: "ghcr.io/datadog/dd-trace-php/dd-library-php"
+  PACKAGE_PATH: "orgs/DataDog/packages/container/dd-trace-php%2Fdd-library-php"
+  S3_BASE: "https://dd-trace-php-builds.s3.us-east-1.amazonaws.com/ci"
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write   # push + delete versions of ghcr.io/datadog/dd-trace-php/dd-library-php
+      contents: read
+    steps:
+      - name: Download build artifacts from S3
+        env:
+          REF_SLUG: ${{ github.event.inputs.ref_slug }}
+        run: |
+          set -euo pipefail
+          base="${S3_BASE}/${REF_SLUG}"
+          mkdir -p packages/amd64 packages/arm64
+          curl -fSL --retry 3 "${base}/dd-library-php-x86_64-linux-gnu.tar.gz"  -o packages/amd64/dd-library-php-x86_64-linux-gnu.tar.gz
+          curl -fSL --retry 3 "${base}/dd-library-php-aarch64-linux-gnu.tar.gz" -o packages/arm64/dd-library-php-aarch64-linux-gnu.tar.gz
+          curl -fSL --retry 3 "${base}/datadog-setup.php" -o packages/amd64/datadog-setup.php
+          cp packages/amd64/datadog-setup.php packages/arm64/datadog-setup.php
+
+      - name: Log in to GHCR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ACTOR: ${{ github.actor }}
+        run: echo "$GH_TOKEN" | docker login ghcr.io -u "$ACTOR" --password-stdin
+
+      - name: Delete any pre-existing image with this tag
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REF_SLUG: ${{ github.event.inputs.ref_slug }}
+        run: | # avoid piling up untagged versions when re-pushing the same tag
+          gh api "${PACKAGE_PATH}/versions" \
+            --jq ".[] | select(.metadata.container.tags[]? == \"${REF_SLUG}\") | .id" \
+            | xargs -r -I {} gh api "${PACKAGE_PATH}/versions/{}" -X DELETE
+          echo "Pre-delete complete for tag ${REF_SLUG}"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
+
+      - name: Build and push image
+        env:
+          REF_SLUG: ${{ github.event.inputs.ref_slug }}
+        run: |
+          set -euo pipefail
+          # FROM scratch + COPY only (no RUN) -> no QEMU needed for the foreign arch.
+          printf 'FROM scratch\nARG TARGETARCH\nCOPY packages/${TARGETARCH}/dd-library-php-*-linux-gnu.tar.gz /\nCOPY packages/${TARGETARCH}/datadog-setup.php /\n' > Dockerfile.system-tests
+          docker buildx build --platform linux/amd64,linux/arm64 \
+            -f Dockerfile.system-tests -t "${IMAGE}:${REF_SLUG}" --push .
+          echo "Pushed ${IMAGE}:${REF_SLUG}"

.gitlab/generate-package.php

@@ -1558,42 +1558,19 @@ function appsec_image_from_tag_mapping(string $tag): string
     paths:
       - packages/datadog-setup.php
 
-# Runs on every non-default branch so system tests can be run against any (non-default) in-progress branch.
-"publish docker image for system tests (token)":
+# System-tests image publishing. Runs on every non-default branch so system tests can be
+# run against any in-progress branch (DataDog/system-tests LIBRARY_TARGET_BRANCH).
+
+"publish docker image for system tests (upload)":
   stage: release
-  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.17.32
   tags: [ "arch:amd64" ]
-  id_tokens:
-    DDOCTOSTS_ID_TOKEN:
-      aud: dd-octo-sts
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
       when: never
     - when: always
   allow_failure: true
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages > github_token_system_tests.txt
-  artifacts:
-    paths:
-      - github_token_system_tests.txt
-    expire_in: 1 hour
-    when: on_success
-
-"publish docker image for system tests":
-  stage: release
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
-  tags: [ "docker-in-docker:amd64" ]
-  resource_group: "publish-system-tests-image-${CI_COMMIT_REF_SLUG}"
-  interruptible: true
-  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      when: never
-    - when: always
   needs:
-    - job: "publish docker image for system tests (token)"
-      artifacts: true
     - job: "datadog-setup.php"
       artifacts: true
     - job: "package extension: [amd64, x86_64-unknown-linux-gnu]"
@@ -1602,74 +1579,57 @@ function appsec_image_from_tag_mapping(string $tag): string
       artifacts: true
   variables:
     GIT_STRATEGY: none
+  script: |
+    set -e
+    # Normalise the arch tarball names so the GitHub Actions side needs only the branch slug.
+    DEST="s3://dd-trace-php-builds/ci/${CI_COMMIT_REF_SLUG}"
+    aws s3 cp packages/dd-library-php-*-x86_64-linux-gnu.tar.gz  "${DEST}/dd-library-php-x86_64-linux-gnu.tar.gz"
+    aws s3 cp packages/dd-library-php-*-aarch64-linux-gnu.tar.gz "${DEST}/dd-library-php-aarch64-linux-gnu.tar.gz"
+    aws s3 cp packages/datadog-setup.php                         "${DEST}/datadog-setup.php"
+    echo "Uploaded system-tests artifacts to ${DEST}/"
+
+"publish docker image for system tests (dispatch)":
+  stage: release
+  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  tags: [ "arch:amd64" ]
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: never
+    - when: always
   allow_failure: true
+  needs:
+    - job: "publish docker image for system tests (upload)"
+  variables:
+    GIT_STRATEGY: none
   script: |
     set -e
-    IMAGE="ghcr.io/datadog/dd-trace-php/dd-library-php:${CI_COMMIT_REF_SLUG}"
-    VERSIONS_URL="https://api.github.com/orgs/DataDog/packages/container/dd-trace-php%2Fdd-library-php/versions"
-    GITHUB_TOKEN=$(<github_token_system_tests.txt)
-
-    apt-get update -qq && apt-get install -y -qq curl jq > /dev/null
+    # octo-sts token: actions:write to dispatch the workflow, pull_requests:read to gate on an open PR.
+    GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/dd-trace-php --policy gitlab-ci-publish-packages)
 
-    STATUS=$(curl -s -o /tmp/pulls.json -w "%{http_code}" -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \
-        "https://api.github.com/repos/DataDog/dd-trace-php/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open")
-    if [ "${STATUS}" != "200" ]; then
-      echo "ERROR: failed to check for an open PR on branch ${CI_COMMIT_BRANCH} (HTTP ${STATUS}):"; cat /tmp/pulls.json
-      exit 1
-    fi
-    PR_COUNT=$(jq 'length' /tmp/pulls.json)
-    if [ "${PR_COUNT}" -eq 0 ]; then
+    # Only publish for branches with an open PR (system tests pull the image per-PR).
+    PR_COUNT=$(curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \
+        "https://api.github.com/repos/DataDog/dd-trace-php/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open" | jq 'length')
+    if [ "${PR_COUNT:-0}" -eq 0 ]; then
       echo "No open PR for branch ${CI_COMMIT_BRANCH}; skipping system-tests image publish."
-      exit 3
+      exit 0
     fi
-    echo "Found open PR(s) on branch ${CI_COMMIT_BRANCH}, continuing"
-
-    docker login ghcr.io -u DataDog --password-stdin < github_token_system_tests.txt
-    echo "Docker login to ghcr.io succeeded"
-
-    gh_api() { curl -s -o /tmp/resp.json -w "%{http_code}" -X "$1" \
-        -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" "$2"; }
-
-    # delete any pre-existing image with the same tag so pushing with the same tag doesn't pile up untagged versions.
-    echo "== Checking token and existing image tagged ${CI_COMMIT_REF_SLUG} =="
-    STATUS=$(gh_api GET "$VERSIONS_URL")
-    if [ "$STATUS" != "200" ]; then
-      echo "WARNING: token check failed (HTTP ${STATUS}), skipping pre-delete:"; cat /tmp/resp.json
-    else
-      echo "Token OK (HTTP 200): listed package versions for dd-trace-php/dd-library-php"
-      VERSION_ID=$(jq -r --arg TAG "${CI_COMMIT_REF_SLUG}" \
-          '.[] | select(.metadata.container.tags[]? == $TAG) | .id' /tmp/resp.json | head -n1)
-      if [ -z "$VERSION_ID" ]; then
-        echo "No pre-existing image tagged ${CI_COMMIT_REF_SLUG} found, nothing to delete"
-      else
-        STATUS=$(gh_api DELETE "${VERSIONS_URL}/${VERSION_ID}")
-        if [ "$STATUS" = "204" ]; then
-          echo "Deleted existing image version ${VERSION_ID} (HTTP 204)"
-        else
-          echo "WARNING: failed to delete existing image version ${VERSION_ID} (HTTP ${STATUS}), continuing anyway:"
-          cat /tmp/resp.json
-        fi
-      fi
+    echo "Found open PR(s) on branch ${CI_COMMIT_BRANCH}, dispatching publish workflow"
+
+    # GHCR write must come from GitHub Actions' GITHUB_TOKEN, so trigger the workflow that
+    # pulls the artifacts back down from S3 and pushes the image.
+    PAYLOAD=$(jq -n --arg ref "${CI_DEFAULT_BRANCH}" --arg slug "${CI_COMMIT_REF_SLUG}" \
+        '{ref: $ref, inputs: {ref_slug: $slug}}')
+    HTTP=$(curl -s -o /tmp/dispatch.json -w "%{http_code}" -X POST \
+        -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" \
+        "https://api.github.com/repos/DataDog/dd-trace-php/actions/workflows/publish-system-tests-image.yml/dispatches" \
+        -d "${PAYLOAD}")
+    if [ "${HTTP}" != "204" ]; then
+      echo "ERROR: workflow_dispatch failed (HTTP ${HTTP}):"; cat /tmp/dispatch.json; exit 1
     fi
-
-    echo "== Building and pushing ${IMAGE} (linux/amd64, linux/arm64) =="
-    # stage each arch's tarball under its own dir so the Dockerfile can pick the right one via buildx's TARGETARCH arg.
-    mkdir -p packages/amd64 packages/arm64
-    cp packages/dd-library-php-*-x86_64-linux-gnu.tar.gz packages/amd64/
-    cp packages/dd-library-php-*-aarch64-linux-gnu.tar.gz packages/arm64/
-    cp packages/datadog-setup.php packages/amd64/
-    cp packages/datadog-setup.php packages/arm64/
-
-    printf 'FROM scratch\nARG TARGETARCH\nCOPY packages/${TARGETARCH}/dd-library-php-*-linux-gnu.tar.gz /\nCOPY packages/${TARGETARCH}/datadog-setup.php /\n' \
-        > Dockerfile.system-tests
-    BUILDER="system-tests-builder-${CI_JOB_ID}"
-    docker buildx create --name "$BUILDER" --driver docker-container
-    docker buildx build --builder "$BUILDER" --platform linux/amd64,linux/arm64 \
-        -f Dockerfile.system-tests -t "$IMAGE" --push .
-    echo "Pushed $IMAGE"
-  after_script:
-    - rm -f github_token_system_tests.txt
-    - docker buildx rm "system-tests-builder-${CI_JOB_ID}" || true
+    echo "Dispatched publish-system-tests-image.yml (tag ${CI_COMMIT_REF_SLUG}) on ${CI_DEFAULT_BRANCH}"
 
 "bundle for reliability env":
   stage: shared-pipeline