ci-images: generate per-version buildx-bake build + mirror pipeline

The image list (PHP versions and tags) is derived from the docker-compose.yml
+ .env files in each dockerfiles/ci/<os>/ dir (single source of truth).
.gitlab/generate-ci-images.php renders .gitlab/ci-images.yml.tpl, emitting per
Linux OS:
  - <OS> build      : one matrix job over PHP version; 'docker buildx bake
                      --no-cache --pull --push' builds both arches (x-bake
                      platforms from compose) on the amd64 runner's managed ci
                      builder and pushes a multi-arch manifest to
                      registry.ddbuild.io
  - <OS> publish:<v>: manual mirror to Docker Hub via DataDog/public-images,
                      dependency-free (just syncs whatever is in the internal
                      registry)

Static preamble + Windows jobs live in .gitlab/ci-images.static.yml (Windows
is single-arch). The generator runs in generate-templates and is triggered as
a child pipeline via the manual 'ci-images' job; the old .gitlab/ci-images.yml
local include is removed.

Author: realFlowControl

.gitlab-ci.yml

@@ -18,7 +18,6 @@ include:
   - project: DataDog/apm-reliability/libdatadog-build
     ref: 5826819695d93286569e70ed087ae6bf906ce2c3
     file: templates/ci_authenticated_job.yml
-  - local: .gitlab/ci-images.yml
 
 generate-templates:
   stage: build
@@ -57,6 +56,7 @@ generate-templates:
     - php ./.gitlab/generate-appsec.php   | tee .gitlab/appsec-gen.yml
     - php ./.gitlab/generate-profiler.php | tee .gitlab/profiler-gen.yml
     - php ./.gitlab/generate-shared.php   | tee .gitlab/shared-gen.yml
+    - php ./.gitlab/generate-ci-images.php | tee .gitlab/ci-images-gen.yml
   variables:
     GIT_SUBMODULE_STRATEGY: none
   artifacts:
@@ -90,6 +90,22 @@ appsec-trigger:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
     GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
 
+# Manual maintenance pipeline that (re)builds the CI Docker images. Generated
+# from dockerfiles/ci/*/docker-compose.yml + .env so versions live in one place.
+# No strategy: depend — the parent must not wait on these manual jobs.
+ci-images:
+  stage: ci-build
+  rules:
+    - when: manual
+      allow_failure: true
+  needs:
+    - job: generate-templates
+      artifacts: true
+  trigger:
+    include:
+      - artifact: .gitlab/ci-images-gen.yml
+        job: generate-templates
+
 profiler-trigger:
   stage: tests
   needs: [ "generate-templates" ]

.gitlab/ci-images.yml .gitlab/ci-images.static.yml.gitlab/ci-images.yml renamed to .gitlab/ci-images.static.yml

@@ -1,99 +1,48 @@
-variables:
-  CI_REGISTRY_IMAGE:
-    value: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci"
+# DO NOT EDIT THE GENERATED LINUX JOBS — they are produced by
+# .gitlab/generate-ci-images.php from the docker-compose.yml + .env files.
+# This file holds the STATIC preamble (stages, templates) and the Windows jobs,
+# which have no multi-arch manifest and stay hand-maintained.
 
-CentOS:
-  stage: ci-build
-  rules:
-    - when: manual
-      allow_failure: true
-  needs: []
-  tags: ["arch:amd64"]
-  timeout: 4h
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
-  variables:
-    DDCI_CONFIGURE_OTEL_EXPORTER: "true"
-  parallel:
-    matrix:
-      - PHP_VERSION:
-        - base
-        - php-8.5
-        - php-8.4
-        - php-8.3
-        - php-8.2
-        - php-8.1
-        - php-8.0
-        - php-7.4
-        - php-7.3
-        - php-7.2
-        - php-7.1
-        - php-7.0
-  script:
-    - cd dockerfiles/ci/centos/7
-    - docker buildx bake --no-cache --pull --push $PHP_VERSION
+stages:
+  - ci-build
+  - ci-publish
 
-Alpine:
+variables:
+  CI_REGISTRY_IMAGE: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci"
+
+.linux_image_build:
   stage: ci-build
   rules:
     - when: manual
       allow_failure: true
   needs: []
-  tags: ["arch:amd64"]
   timeout: 4h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
   variables:
     DDCI_CONFIGURE_OTEL_EXPORTER: "true"
-  parallel:
-    matrix:
-      - PHP_VERSION:
-        - base-alpine
-        - 8.5-alpine
-        - 8.4-alpine
-        - 8.3-alpine
-        - 8.2-alpine
-        - 8.1-alpine
-        - 8.0-alpine
-        - 7.4-alpine
-        - 7.3-alpine
-        - 7.2-alpine
-        - 7.1-alpine
-        - 7.0-alpine
-  script:
-    - cd dockerfiles/ci/alpine_compile_extension
-    - docker buildx bake --no-cache --pull --push $PHP_VERSION
+    KUBERNETES_CPU_REQUEST: "8"
+    KUBERNETES_CPU_LIMIT: "8"
+    KUBERNETES_MEMORY_REQUEST: "16Gi"
+    KUBERNETES_MEMORY_LIMIT: "16Gi"
+    MAKE_JOBS: "$KUBERNETES_CPU_LIMIT"
 
-Bookworm:
-  stage: ci-build
+.linux_publish:
+  stage: ci-publish
   rules:
     - when: manual
       allow_failure: true
+  # No deps: a publish just mirrors whatever already exists in
+  # registry.ddbuild.io to Docker Hub, so it can run without (re)building.
   needs: []
-  tags: ["arch:amd64"]
-  timeout: 4h
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
+  trigger:
+    project: DataDog/public-images
+    branch: main
+  # $TAG is supplied per matrix entry by the generated publish jobs.
   variables:
-    DDCI_CONFIGURE_OTEL_EXPORTER: "true"
-    MAKE_JOBS: "2"
-  parallel:
-    matrix:
-      - PHP_VERSION:
-        - base
-        - php-8.5
-        - php-8.4
-        - php-8.3
-        - php-8.2
-        - php-8.1
-        - php-8.0
-        - php-8.0-shared-ext
-        - php-7.4
-        - php-7.4-shared-ext
-        - php-7.3
-        - php-7.2
-        - php-7.1
-        - php-7.0
-  script:
-    - cd dockerfiles/ci/bookworm
-    - docker buildx bake --no-cache --pull --push $PHP_VERSION
+    IMG_REGISTRIES: "dockerhub"
+    IMG_SIGNING: false
+    IMG_SOURCES: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:${TAG}"
+    IMG_DESTINATIONS: "dd-trace-ci:${TAG}"
 
 .windows_image_build:
   stage: ci-build
@@ -196,98 +145,6 @@ Bookworm:
         - "php-7.3"
         - "php-7.2"
 
-Publish CentOS:
-  stage: ci-publish
-  rules:
-    - when: manual
-      allow_failure: true
-  needs:
-    - job: CentOS
-  trigger:
-    project: DataDog/public-images
-    branch: main
-  parallel:
-    matrix:
-      - TAG_NAME:
-        - "centos-7"
-        - "php-8.5_centos-7"
-        - "php-8.4_centos-7"
-        - "php-8.3_centos-7"
-        - "php-8.2_centos-7"
-        - "php-8.1_centos-7"
-        - "php-8.0_centos-7"
-        - "php-7.4_centos-7"
-        - "php-7.3_centos-7"
-        - "php-7.2_centos-7"
-        - "php-7.1_centos-7"
-        - "php-7.0_centos-7"
-  variables:
-    IMG_SOURCES: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:${TAG_NAME}"
-    IMG_DESTINATIONS: "dd-trace-ci:${TAG_NAME}"
-    IMG_REGISTRIES: "dockerhub"
-
-Publish Bookworm:
-  stage: ci-publish
-  rules:
-    - when: manual
-      allow_failure: true
-  needs:
-    - job: Bookworm
-  trigger:
-    project: DataDog/public-images
-    branch: main
-  parallel:
-    matrix:
-      - TAG_NAME:
-        - "bookworm-9"
-        - "php-8.5_bookworm-9"
-        - "php-8.4_bookworm-9"
-        - "php-8.3_bookworm-9"
-        - "php-8.2_bookworm-9"
-        - "php-8.1_bookworm-9"
-        - "php-8.0_bookworm-9"
-        - "php-8.0-shared-ext-9"
-        - "php-7.4_bookworm-9"
-        - "php-7.4-shared-ext-9"
-        - "php-7.3_bookworm-9"
-        - "php-7.2_bookworm-9"
-        - "php-7.1_bookworm-9"
-        - "php-7.0_bookworm-9"
-  variables:
-    IMG_SOURCES: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:${TAG_NAME}"
-    IMG_DESTINATIONS: "dd-trace-ci:${TAG_NAME}"
-    IMG_REGISTRIES: "dockerhub"
-
-Publish Alpine:
-  stage: ci-publish
-  rules:
-    - when: manual
-      allow_failure: true
-  needs:
-    - job: Alpine
-  trigger:
-    project: DataDog/public-images
-    branch: main
-  parallel:
-    matrix:
-      - TAG_NAME:
-        - "php-compile-extension-alpine"
-        - "php-compile-extension-alpine-8.5"
-        - "php-compile-extension-alpine-8.4"
-        - "php-compile-extension-alpine-8.3"
-        - "php-compile-extension-alpine-8.2"
-        - "php-compile-extension-alpine-8.1"
-        - "php-compile-extension-alpine-8.0"
-        - "php-compile-extension-alpine-7.4"
-        - "php-compile-extension-alpine-7.3"
-        - "php-compile-extension-alpine-7.2"
-        - "php-compile-extension-alpine-7.1"
-        - "php-compile-extension-alpine-7.0"
-  variables:
-    IMG_SOURCES: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:${TAG_NAME}"
-    IMG_DESTINATIONS: "dd-trace-ci:${TAG_NAME}"
-    IMG_REGISTRIES: "dockerhub"
-
 Publish Windows:
   stage: ci-publish
   rules:
@@ -321,3 +178,4 @@ Publish Windows:
     IMG_SOURCES: "registry.ddbuild.io/ci/dd-trace-php/dd-trace-ci:${TAG_NAME}"
     IMG_DESTINATIONS: "dd-trace-ci:${TAG_NAME}"
     IMG_REGISTRIES: "dockerhub"
+    IMG_SIGNING: false

.gitlab/ci-images.yml.tpl

@@ -0,0 +1,37 @@
+<?php echo file_get_contents(__DIR__ . '/ci-images.static.yml'); ?>
+<?php foreach ($osList as ['name' => $os, 'dir' => $dir, 'services' => $services]): ?>
+<?php /*
+  One build job per PHP version. buildx bake reads the per-service x-bake
+  platforms from docker-compose.yml and builds BOTH arches on the amd64 runner's
+  managed "ci" builder, pushing a multi-arch manifest straight to the tag in the
+  compose `image:` field. No per-arch split, no manifest fuse job.
+*/ ?>
+
+<?= $os ?> build:
+  extends: .linux_image_build
+  tags: ["arch:amd64"]
+  parallel:
+    matrix:
+      - PHP_VERSION:
+<?php foreach (array_keys($services) as $svc): ?>
+          - <?= $svc, "\n" ?>
+<?php endforeach; ?>
+  script:
+    - cd <?= $dir, "\n" ?>
+    - docker buildx bake --no-cache --pull --push "${PHP_VERSION}"
+<?php /*
+  Mirror to Docker Hub, one matrix job per OS (grouped in the UI like the
+  builds). Independent (needs: [] via .linux_publish): just syncs whatever is
+  already in registry.ddbuild.io, so it can run without rebuilding. IMG_SOURCES
+  / IMG_DESTINATIONS are built from $TAG in .linux_publish.
+*/ ?>
+
+<?= $os ?> publish:
+  extends: .linux_publish
+  parallel:
+    matrix:
+      - TAG:
+<?php foreach ($services as $tag): ?>
+          - "<?= $tag ?>"
+<?php endforeach; ?>
+<?php endforeach; ?>

.gitlab/generate-ci-images.php

@@ -0,0 +1,87 @@
+<?php
+
+/**
+ * Generates the CI image build/manifest/publish pipeline from ci-images.yml.tpl.
+ *
+ * Source of truth (NO duplication):
+ *   - dockerfiles/ci/<os>/docker-compose.yml : service name -> image:TAG
+ *   - dockerfiles/ci/bookworm/.env           : $BOOKWORM_NEXT_VERSION etc.
+ *
+ * The compose service name is the `docker buildx bake` target and the build
+ * matrix value; the `image:` tag (with env vars resolved) is the published tag.
+ * Per Linux image the template emits one build matrix job over PHP versions
+ * (bake builds the multi-arch image and pushes it) plus a manual mirror/publish
+ * job per service. The static preamble (templates) and Windows jobs live in
+ * ci-images.static.yml (Windows is single-arch).
+ */
+
+$root = dirname(__DIR__);
+
+// Resolve $VAR / ${VAR} from a key=value .env file.
+function parse_env(string $path): array
+{
+    $env = [];
+    foreach (@file($path, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
+        if (preg_match('/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/', $line, $m)) {
+            $env[$m[1]] = $m[2];
+        }
+    }
+    return $env;
+}
+
+function substitute(string $s, array $env): string
+{
+    return preg_replace_callback('/\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?/', function ($m) use ($env) {
+        return $env[$m[1]] ?? $m[0];
+    }, $s);
+}
+
+// Parse a docker-compose.yml into [service => tag], preserving file order.
+function parse_compose(string $path, array $env): array
+{
+    $services = [];
+    $cur = null;
+    $inServices = false;
+    foreach (file($path, FILE_IGNORE_NEW_LINES) as $line) {
+        if (preg_match('/^services:\s*$/', $line)) {
+            $inServices = true;
+            continue;
+        }
+        if (!$inServices) {
+            continue;
+        }
+        if (preg_match('/^\S/', $line)) { // back to a top-level key
+            $inServices = false;
+            continue;
+        }
+        if (preg_match('/^  ([A-Za-z0-9][A-Za-z0-9._-]*):\s*$/', $line, $m)) {
+            $cur = $m[1];
+            $services[$cur] = null;
+            continue;
+        }
+        // image: ${CI_REGISTRY_IMAGE:-...}:TAG   (first image: wins per service)
+        if ($cur !== null && $services[$cur] === null
+            && preg_match('/^    image:\s*[\'"]?\$\{[^}]+\}:([^\s\'"]+)/', $line, $m)) {
+            $services[$cur] = substitute($m[1], $env);
+        }
+    }
+    return array_filter($services, fn($v) => $v !== null);
+}
+
+$dirs = [
+    "Bookworm" => "dockerfiles/ci/bookworm",
+    "CentOS"   => "dockerfiles/ci/centos/7",
+    "Alpine"   => "dockerfiles/ci/alpine_compile_extension",
+];
+
+$osList = [];
+foreach ($dirs as $os => $dir) {
+    $services = parse_compose("$root/$dir/docker-compose.yml", parse_env("$root/$dir/.env"));
+    if (!$services) {
+        fwrite(STDERR, "WARNING: no services parsed for $os ($dir)\n");
+        continue;
+    }
+    $osList[] = ["name" => $os, "dir" => $dir, "services" => $services];
+}
+
+require __DIR__ . "/ci-images.yml.tpl";