From ef6a8e18306d6541d766fbb468f7d691d97ed548 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 17:33:05 +0200 Subject: [PATCH 1/7] feat: declare base image build dependencies in weblog_metadata.yml --- docs/understand/weblogs/weblog-metadata.md | 20 +++ utils/_context/weblog_metadata.py | 17 +++ utils/build/docker/nodejs/weblog_metadata.yml | 23 ++++ utils/build/docker/php/weblog_metadata.yml | 114 ++++++++++++++++++ utils/build/docker/python/weblog_metadata.yml | 18 +++ 5 files changed, 192 insertions(+) create mode 100644 utils/build/docker/php/weblog_metadata.yml diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md index 2730d9e6875..7a121c3131b 100644 --- a/docs/understand/weblogs/weblog-metadata.md +++ b/docs/understand/weblogs/weblog-metadata.md @@ -45,3 +45,23 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`. `WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges: 1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata). 2. Explicit overrides from `weblog_metadata.yml`. + +## Base image dependencies + +A `weblog_metadata.yml` may also declare a top-level `base_image_dependencies` section, unrelated +to the per-weblog entries above, used by the `build_base_images` CI job +(`utils/scripts/build_base_images.py`) to know when a weblog base image needs to be rebuilt: + +```yaml +base_image_dependencies: + : + - + - ... +``` + +For each target listed there, the job computes a content hash from the resolved +`docker-bake.hcl` target config, the target's Dockerfile, and every git-tracked file under the +listed paths, then pushes the base image to Docker Hub tagged `-` if that tag +doesn't already exist. It never overwrites an existing tag, so weblog Dockerfiles that `FROM` a +base image must have their tag updated by hand after a new one is pushed (run the script with +`--dry-run` to find the current tag for each target). diff --git a/utils/_context/weblog_metadata.py b/utils/_context/weblog_metadata.py index 7d72d1a2bfb..5523d712053 100644 --- a/utils/_context/weblog_metadata.py +++ b/utils/_context/weblog_metadata.py @@ -68,8 +68,25 @@ def _load_explicit_metadata(library: str) -> dict[str, "WeblogMetaData"]: with path.open() as f: data: dict = yaml.safe_load(f) or {} + data.pop("base_image_dependencies", None) + return {name: WeblogMetaData(name=name, library=library, **kwargs) for name, kwargs in data.items()} + @staticmethod + def load_base_image_dependencies(library: str) -> dict[str, list[str]]: + """Returns the `base_image_dependencies` section of weblog_metadata.yml: a mapping of + docker-bake.hcl target name to the list of paths (files or directories) that base image + depends on, used to compute a content-hash tag for the base image build job. + """ + path = Path(f"utils/build/docker/{library}/weblog_metadata.yml") + if not path.exists(): + return {} + + with path.open() as f: + data: dict = yaml.safe_load(f) or {} + + return data.get("base_image_dependencies", {}) or {} + @staticmethod def load(library: str) -> list["WeblogMetaData"]: metadata = WeblogMetaData._load_explicit_metadata(library) diff --git a/utils/build/docker/nodejs/weblog_metadata.yml b/utils/build/docker/nodejs/weblog_metadata.yml index 802e518616e..4a7da1a482b 100644 --- a/utils/build/docker/nodejs/weblog_metadata.yml +++ b/utils/build/docker/nodejs/weblog_metadata.yml @@ -21,3 +21,26 @@ anthropic-js: google_genai-js: build_mode: none framework_versions: ["1.34.0"] + +# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by +# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. +base_image_dependencies: + express4: + - utils/build/docker/nodejs/express + - utils/build/docker/nodejs/express4/package.json + - utils/build/docker/nodejs/express4/bun.lock + - utils/build/docker/nodejs/nft-prune.mjs + express5: + - utils/build/docker/nodejs/express + - utils/build/docker/nodejs/express5/package.json + - utils/build/docker/nodejs/express5/bun.lock + - utils/build/docker/nodejs/nft-prune.mjs + express4-typescript: + - utils/build/docker/nodejs/express4-typescript + - utils/build/docker/nodejs/nft-prune.mjs + fastify: + - utils/build/docker/nodejs/fastify + - utils/build/docker/nodejs/nft-prune.mjs + nextjs: + - utils/build/docker/nodejs/nextjs + - utils/build/docker/nodejs/nft-prune.mjs diff --git a/utils/build/docker/php/weblog_metadata.yml b/utils/build/docker/php/weblog_metadata.yml new file mode 100644 index 00000000000..91ffc08aa5d --- /dev/null +++ b/utils/build/docker/php/weblog_metadata.yml @@ -0,0 +1,114 @@ +# docs/understand/weblogs/weblog-metadata.md + +# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by +# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. +base_image_dependencies: + apache-mod-7_0: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_1: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_2: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_3: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_4: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_0: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_1: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_2: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_0-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_1-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_2-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_3-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-7_4-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_0-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_1-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + apache-mod-8_2-zts: + - utils/build/docker/php/apache-mod + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_0: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_1: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_2: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_3: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-7_4: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_0: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_1: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_2: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common + php-fpm-8_5: + - utils/build/docker/php/php-fpm + - utils/build/docker/php/apt-sources.d + - utils/build/docker/php/weblogs + - utils/build/docker/php/common diff --git a/utils/build/docker/python/weblog_metadata.yml b/utils/build/docker/python/weblog_metadata.yml index c0b67e68fa7..c8e71bb0297 100644 --- a/utils/build/docker/python/weblog_metadata.yml +++ b/utils/build/docker/python/weblog_metadata.yml @@ -9,3 +9,21 @@ anthropic-py: google_genai-py: build_mode: none framework_versions: ["1.55.0"] + +# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by +# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md. +base_image_dependencies: + django-poc: + - utils/build/docker/python/django/requirements-django-poc.txt + django-py3_13: + - utils/build/docker/python/django/requirements-django-py3.13.txt + python3_12: + - utils/build/docker/python/django/requirements-python3.12.txt + fastapi: + - utils/build/docker/python/fastapi/requirements-fastapi.txt + flask-poc: + - utils/build/docker/python/flask/requirements-flask-poc.txt + uwsgi-poc: + - utils/build/docker/python/flask/requirements-uwsgi-poc.txt + tornado: + - utils/build/docker/python/tornado/requirements-tornado.txt From 8dccc014e6757408ae61c9206f97f3b1209b314e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 17:33:15 +0200 Subject: [PATCH 2/7] refactor: drop static -vN suffix from base image bake tags Base images will now be tagged with a content hash appended by utils/scripts/build_base_images.py. --- utils/build/docker/nodejs/docker-bake.hcl | 10 ++--- utils/build/docker/php/docker-bake.hcl | 50 +++++++++++------------ utils/build/docker/python/docker-bake.hcl | 14 +++---- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/utils/build/docker/nodejs/docker-bake.hcl b/utils/build/docker/nodejs/docker-bake.hcl index a8430a0ed51..45c1d071caf 100644 --- a/utils/build/docker/nodejs/docker-bake.hcl +++ b/utils/build/docker/nodejs/docker-bake.hcl @@ -18,33 +18,33 @@ target "express4" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/express4.base.Dockerfile" - tags = ["datadog/system-tests:express4.base-v3"] + tags = ["datadog/system-tests:express4.base"] } target "express5" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/express5.base.Dockerfile" - tags = ["datadog/system-tests:express5.base-v3"] + tags = ["datadog/system-tests:express5.base"] } target "fastify" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/fastify.base.Dockerfile" - tags = ["datadog/system-tests:fastify.base-v3"] + tags = ["datadog/system-tests:fastify.base"] } target "express4-typescript" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/express4-typescript.base.Dockerfile" - tags = ["datadog/system-tests:express4-typescript.base-v3"] + tags = ["datadog/system-tests:express4-typescript.base"] } target "nextjs" { inherits = ["_common"] context = "." dockerfile = "utils/build/docker/nodejs/nextjs.base.Dockerfile" - tags = ["datadog/system-tests:nextjs.base-v3"] + tags = ["datadog/system-tests:nextjs.base"] } diff --git a/utils/build/docker/php/docker-bake.hcl b/utils/build/docker/php/docker-bake.hcl index 880178b38b9..47308716f3e 100644 --- a/utils/build/docker/php/docker-bake.hcl +++ b/utils/build/docker/php/docker-bake.hcl @@ -35,158 +35,158 @@ group "default" { target "apache-mod-7_0" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.0.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.0.base"] } target "apache-mod-7_1" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.1.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.1.base"] } target "apache-mod-7_2" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.2.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.2.base"] } target "apache-mod-7_3" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.3.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.3.base"] } target "apache-mod-7_4" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-7.4.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.4.base"] } target "apache-mod-8_0" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.0.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.0.base"] } target "apache-mod-8_1" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.1.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.1.base"] } target "apache-mod-8_2" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release" } - tags = ["datadog/system-tests:apache-mod-8.2.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.2.base"] } target "apache-mod-7_0-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.0", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.0-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.0-zts.base"] } target "apache-mod-7_1-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.1", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.1-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.1-zts.base"] } target "apache-mod-7_2-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.2", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.2-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.2-zts.base"] } target "apache-mod-7_3-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.3", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.3-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.3-zts.base"] } target "apache-mod-7_4-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "7.4", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-7.4-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-7.4-zts.base"] } target "apache-mod-8_0-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.0", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.0-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.0-zts.base"] } target "apache-mod-8_1-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.1", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.1-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.1-zts.base"] } target "apache-mod-8_2-zts" { dockerfile = "utils/build/docker/php/apache-mod.base.Dockerfile" args = { PHP_VERSION = "8.2", VARIANT = "release-zts" } - tags = ["datadog/system-tests:apache-mod-8.2-zts.base-v1"] + tags = ["datadog/system-tests:apache-mod-8.2-zts.base"] } target "php-fpm-7_0" { context = "utils/build/docker/php/" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.0" } - tags = ["datadog/system-tests:php-fpm-7.0.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.0.base"] } target "php-fpm-7_1" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.1" } - tags = ["datadog/system-tests:php-fpm-7.1.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.1.base"] } target "php-fpm-7_2" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.2" } - tags = ["datadog/system-tests:php-fpm-7.2.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.2.base"] } target "php-fpm-7_3" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.3" } - tags = ["datadog/system-tests:php-fpm-7.3.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.3.base"] } target "php-fpm-7_4" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "7.4" } - tags = ["datadog/system-tests:php-fpm-7.4.base-v1"] + tags = ["datadog/system-tests:php-fpm-7.4.base"] } target "php-fpm-8_0" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.0" } - tags = ["datadog/system-tests:php-fpm-8.0.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.0.base"] } target "php-fpm-8_1" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.1" } - tags = ["datadog/system-tests:php-fpm-8.1.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.1.base"] } target "php-fpm-8_2" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.2" } - tags = ["datadog/system-tests:php-fpm-8.2.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.2.base"] } target "php-fpm-8_5" { context = "utils/build/docker/php" dockerfile = "php-fpm.base.Dockerfile" args = { PHP_VERSION = "8.5" } - tags = ["datadog/system-tests:php-fpm-8.5.base-v1"] + tags = ["datadog/system-tests:php-fpm-8.5.base"] } diff --git a/utils/build/docker/python/docker-bake.hcl b/utils/build/docker/python/docker-bake.hcl index 3e3f0e8af99..0cf888f82d1 100644 --- a/utils/build/docker/python/docker-bake.hcl +++ b/utils/build/docker/python/docker-bake.hcl @@ -15,41 +15,41 @@ group "default" { target "django-py3_13" { context = "." dockerfile = "utils/build/docker/python/django-py3.13.base.Dockerfile" - tags = ["datadog/system-tests:django-py3.13.base-v10"] + tags = ["datadog/system-tests:django-py3.13.base"] } target "fastapi" { context = "." dockerfile = "utils/build/docker/python/fastapi.base.Dockerfile" - tags = ["datadog/system-tests:fastapi.base-v9"] + tags = ["datadog/system-tests:fastapi.base"] } target "python3_12" { context = "." dockerfile = "utils/build/docker/python/python3.12.base.Dockerfile" - tags = ["datadog/system-tests:python3.12.base-v13"] + tags = ["datadog/system-tests:python3.12.base"] } target "django-poc" { context = "." dockerfile = "utils/build/docker/python/django-poc.base.Dockerfile" - tags = ["datadog/system-tests:django-poc.base-v11"] + tags = ["datadog/system-tests:django-poc.base"] } target "flask-poc" { context = "." dockerfile = "utils/build/docker/python/flask-poc.base.Dockerfile" - tags = ["datadog/system-tests:flask-poc.base-v14"] + tags = ["datadog/system-tests:flask-poc.base"] } target "uwsgi-poc" { context = "." dockerfile = "utils/build/docker/python/uwsgi-poc.base.Dockerfile" - tags = ["datadog/system-tests:uwsgi-poc.base-v10"] + tags = ["datadog/system-tests:uwsgi-poc.base"] } target "tornado" { context = "." dockerfile = "utils/build/docker/python/tornado.base.Dockerfile" - tags = ["datadog/system-tests:tornado.base-v2"] + tags = ["datadog/system-tests:tornado.base"] } From f4aa6a13de0de1b601ac60b69f7d39c5c3999809 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 18:16:56 +0200 Subject: [PATCH 3/7] feat: add CI job to build and push base images on change Adds build_base_images.py, which for every docker-bake.hcl target listed under base_image_dependencies in a library's weblog_metadata.yml, hashes the target's bake config, Dockerfile, and declared dependencies, and pushes the base image to Docker Hub tagged with that hash if missing. Wires it up as a build_base_images job in .gitlab-ci.yml, running on every push. It is idempotent: existing tags are never overwritten, so it is safe to run ahead of merge. --- .gitlab-ci.yml | 33 ++++++ utils/scripts/build_base_images.py | 170 +++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 utils/scripts/build_base_images.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c92374ff41..67c2ccbc58d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -473,3 +473,36 @@ mirror_images: - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' + +# ────────────────────────────────────────────── +# Rebuild weblog base images whose content changed. +# +# For every library with a `base_image_dependencies` section in +# utils/build/docker//weblog_metadata.yml, computes a content-hash tag per +# docker-bake.hcl target and pushes it if missing (see utils/scripts/build_base_images.py). +# Runs on every push, on every branch: it is idempotent (only pushes new tags, never +# overwrites existing ones), so it is safe to build ahead of merge. Since new tags never +# replace old ones, a weblog Dockerfile's FROM clause must be updated by hand to pick up +# a newly pushed base image (run the script with --dry-run to find the new tag). +# ────────────────────────────────────────────── + +build_base_images: + image: $CI_IMAGE + tags: + - docker-in-docker:amd64 + needs: + - job: build_ci_image + artifacts: false + optional: true + stage: system-tests-utils + allow_failure: true + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ln -sf /system-tests/venv venv + - source venv/bin/activate + script: + - python utils/scripts/build_base_images.py + rules: + - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py new file mode 100644 index 00000000000..4419fa903ff --- /dev/null +++ b/utils/scripts/build_base_images.py @@ -0,0 +1,170 @@ +"""Rebuild and push weblog base images whose content-hash tag is missing from Docker Hub. + +Run from a system-tests checkout, with the runner venv active: + + python utils/scripts/build_base_images.py + +For every library with a `utils/build/docker//weblog_metadata.yml` +declaring a `base_image_dependencies` section, and for every docker-bake.hcl target +listed there: + + 1. Resolve the target's bake config (context/dockerfile/args) via + `docker buildx bake --print`. + 2. Compute a content hash from: the resolved bake config (tags excluded), the + content of the target's Dockerfile, and the content of every git-tracked file + under the paths listed in `base_image_dependencies`. + 3. Take the base tag declared in the bake file (e.g. "datadog/system-tests:express4.base") + and append "-" to get the final tag. + 4. Skip the build if that tag already exists on Docker Hub (`docker manifest inspect`); + otherwise build and push it with that tag. + +This is idempotent and safe to run on every push: it never overwrites an existing +tag, it only creates new ones when the relevant files change. + +Pass --dry-run to only print the computed tag and whether it already exists, +without ever building or pushing (useful to find the tag to put in a weblog +Dockerfile's FROM clause after dependencies change). +""" + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +# Make `utils` importable and resolve paths regardless of the caller's cwd, so a +# plain `python utils/scripts/build_base_images.py` works. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from utils._context.weblog_metadata import WeblogMetaData # noqa: E402 +from utils.const import COMPONENT_GROUPS # noqa: E402 + + +def _bake_file(library: str) -> Path: + return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" + + +def _bake_config(bake_file: Path, target: str) -> dict: + """Resolved bake config (context, dockerfile, args, tags) for a single target.""" + result = subprocess.run( + ["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout)["target"][target] + + +def _tracked_files(path: str) -> list[Path]: + """Every git-tracked file under `path` (a file or a directory), sorted.""" + result = subprocess.run( + ["git", "ls-files", path], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + return sorted(REPO_ROOT / line for line in result.stdout.splitlines() if line) + + +def compute_hash(bake_config: dict, dependencies: list[str]) -> str: + """Content hash for a base image target: bake config (minus tags) + Dockerfile + content + content of every git-tracked file under `dependencies`. + """ + digest = hashlib.sha256() + + config_without_tags = {k: v for k, v in bake_config.items() if k != "tags"} + digest.update(json.dumps(config_without_tags, sort_keys=True).encode()) + + dockerfile = REPO_ROOT / bake_config["context"] / bake_config["dockerfile"] + digest.update(dockerfile.read_bytes()) + + files: list[Path] = [] + for dep in dependencies: + files.extend(_tracked_files(dep)) + + for file in sorted(set(files)): + digest.update(str(file.relative_to(REPO_ROOT)).encode()) + digest.update(file.read_bytes()) + + return digest.hexdigest()[:12] + + +def image_exists(tag: str) -> bool: + result = subprocess.run( + ["docker", "manifest", "inspect", tag], + cwd=REPO_ROOT, + check=False, + capture_output=True, + ) + return result.returncode == 0 + + +def build_and_push(bake_file: Path, target: str, tag: str) -> None: + print(f"Building and pushing {tag}") + subprocess.run( + [ + "docker", + "buildx", + "bake", + "--push", + "--progress=plain", + "--set", + f"{target}.tags={tag}", + "-f", + str(bake_file), + target, + ], + cwd=REPO_ROOT, + check=True, + ) + + +def process_library(library: str, *, dry_run: bool) -> None: + dependencies_by_target = WeblogMetaData.load_base_image_dependencies(library) + if not dependencies_by_target: + return + + bake_file = _bake_file(library) + if not bake_file.exists(): + print(f"Warning: {library} declares base_image_dependencies but has no docker-bake.hcl, skipping") + return + + for target, dependencies in dependencies_by_target.items(): + bake_config = _bake_config(bake_file, target) + base_tag = bake_config["tags"][0] + content_hash = compute_hash(bake_config, dependencies) + tag = f"{base_tag}-{content_hash}" + + if dry_run: + state = "exists" if image_exists(tag) else "missing" + print(f"{library}/{target}: {tag} ({state})") + continue + + if image_exists(tag): + print(f"{tag} already exists, skipping") + continue + + build_and_push(bake_file, target, tag) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Rebuild and push weblog base images with a content-hash tag") + parser.add_argument("--library", help="Only process this library (default: all libraries)") + parser.add_argument( + "--dry-run", + action="store_true", + help="Only print the computed tag and whether it exists on Docker Hub; never build or push", + ) + args = parser.parse_args() + + libraries = [args.library] if args.library else sorted(COMPONENT_GROUPS.all) + for library in libraries: + process_library(library, dry_run=args.dry_run) + + +if __name__ == "__main__": + main() From 611f535c855518a8fdd04679959d16771a04202f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 18:23:34 +0200 Subject: [PATCH 4/7] Waiting for base image build for e2e gitlab --- .gitlab-ci.yml | 3 +-- utils/ci/gitlab/main.yml | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 67c2ccbc58d..9786039e299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -494,8 +494,7 @@ build_base_images: - job: build_ci_image artifacts: false optional: true - stage: system-tests-utils - allow_failure: true + stage: e2e before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 2c596c89566..475fcf4e638 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -155,6 +155,9 @@ system_tests_build_pipeline: - job: mirror_images optional: true artifacts: false + - job: build_base_images + optional: true + artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$SYSTEM_TESTS_FORCE_EXECUTE" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$SYSTEM_TESTS_SKIP_EMPTY_SCENARIO" From 0beb1871acf8b98c4a9b10776d1db8db1823a620 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 19:54:37 +0200 Subject: [PATCH 5/7] fix: surface docker/git command errors in build_base_images.py Previously image_exists() silently treated any docker manifest inspect failure as "tag doesn't exist", masking real errors (auth, network). Print stderr on failure there, and in a new _run() helper used by the other docker/git subprocess calls, so failures are visible instead of only raising a bare CalledProcessError. --- utils/scripts/build_base_images.py | 42 +++++++++++++++++------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/utils/scripts/build_base_images.py b/utils/scripts/build_base_images.py index 4419fa903ff..844d58eaebe 100644 --- a/utils/scripts/build_base_images.py +++ b/utils/scripts/build_base_images.py @@ -46,27 +46,28 @@ def _bake_file(library: str) -> Path: return REPO_ROOT / "utils" / "build" / "docker" / library / "docker-bake.hcl" +def _run(cmd: list[str]) -> subprocess.CompletedProcess: + """Run `cmd`, printing stderr (and stdout, if any) before raising on failure.""" + result = subprocess.run(cmd, cwd=REPO_ROOT, check=False, capture_output=True, text=True) + if result.returncode != 0: + print(f"Error: command failed: {' '.join(cmd)}") + if result.stdout: + print(result.stdout) + if result.stderr: + print(result.stderr) + result.check_returncode() + return result + + def _bake_config(bake_file: Path, target: str) -> dict: """Resolved bake config (context, dockerfile, args, tags) for a single target.""" - result = subprocess.run( - ["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target], - cwd=REPO_ROOT, - check=True, - capture_output=True, - text=True, - ) + result = _run(["docker", "buildx", "bake", "--print", "--progress", "quiet", "-f", str(bake_file), target]) return json.loads(result.stdout)["target"][target] def _tracked_files(path: str) -> list[Path]: """Every git-tracked file under `path` (a file or a directory), sorted.""" - result = subprocess.run( - ["git", "ls-files", path], - cwd=REPO_ROOT, - check=True, - capture_output=True, - text=True, - ) + result = _run(["git", "ls-files", path]) return sorted(REPO_ROOT / line for line in result.stdout.splitlines() if line) @@ -94,18 +95,25 @@ def compute_hash(bake_config: dict, dependencies: list[str]) -> str: def image_exists(tag: str) -> bool: + """Whether `tag` exists on the registry. `docker manifest inspect` exits non-zero both + when the tag genuinely doesn't exist and on unrelated failures (auth, network); print + the error either way so a real failure isn't silently mistaken for a missing tag. + """ result = subprocess.run( ["docker", "manifest", "inspect", tag], cwd=REPO_ROOT, check=False, capture_output=True, + text=True, ) + if result.returncode != 0 and result.stderr: + print(result.stderr.strip()) return result.returncode == 0 def build_and_push(bake_file: Path, target: str, tag: str) -> None: print(f"Building and pushing {tag}") - subprocess.run( + _run( [ "docker", "buildx", @@ -117,9 +125,7 @@ def build_and_push(bake_file: Path, target: str, tag: str) -> None: "-f", str(bake_file), target, - ], - cwd=REPO_ROOT, - check=True, + ] ) From 0797a090cd7d132ce298bdffb2b692983140a482 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 6 Jul 2026 20:26:55 +0200 Subject: [PATCH 6/7] fix(php): install mbstring extension in php-fpm base images composer install failed for php-fpm targets because stripe/stripe-php requires ext-mbstring, which wasn't installed. apache-mod targets were unaffected since they use the official PHP image, which bundles it. --- utils/build/docker/php/php-fpm/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/docker/php/php-fpm/build.sh b/utils/build/docker/php/php-fpm/build.sh index c73bd0d6f3e..8a00683520c 100644 --- a/utils/build/docker/php/php-fpm/build.sh +++ b/utils/build/docker/php/php-fpm/build.sh @@ -31,7 +31,7 @@ nala update printf '#!/bin/sh\n\nexit 101\n' > /usr/sbin/policy-rc.d chmod +x /usr/sbin/policy-rc.d -nala install -y --no-install-recommends tzdata publicsuffix curl apache2 libapache2-mod-fcgid jq ca-certificates git unzip php$PHP_VERSION-fpm php$PHP_VERSION-curl apache2 php$PHP_VERSION-mysql php$PHP_VERSION-pgsql php$PHP_VERSION-xml php$PHP_VERSION-mongodb +nala install -y --no-install-recommends tzdata publicsuffix curl apache2 libapache2-mod-fcgid jq ca-certificates git unzip php$PHP_VERSION-fpm php$PHP_VERSION-curl apache2 php$PHP_VERSION-mysql php$PHP_VERSION-pgsql php$PHP_VERSION-xml php$PHP_VERSION-mongodb php$PHP_VERSION-mbstring rm -rf /usr/sbin/policy-rc.d rm -rf /var/lib/apt/lists/* From b8f1ceb4edb50e131d52d5b0c69f6b9ecb18b8a9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 7 Jul 2026 13:48:46 +0200 Subject: [PATCH 7/7] feat: replace GitHub weblog base image build with a poll for GitLab's push GitHub Actions has no dependency mechanism on GitLab CI jobs, so instead of building weblog base images itself (via the removed, already-broken utils/script/build_base_image.py and PR labels), it now waits for the tag declared in the weblog's Dockerfile to be pushed by GitLab's build_base_images job, polling Docker Hub every 30s with a 15 minute timeout (utils/scripts/wait_for_base_image.py). Drops the _build_weblog_base_images label-gated flag entirely: waiting is cheap (a single manifest inspect when the tag already exists) so it now runs unconditionally for every weblog that has a base image tag. --- .github/workflows/ci.yml | 1 - .../workflows/compute-workflow-parameters.yml | 7 -- .github/workflows/run-end-to-end.yml | 8 +- .github/workflows/system-tests.yml | 14 +--- docs/CI/github-actions.md | 1 - docs/edit/update-docker-images.md | 25 ++++-- docs/understand/weblogs/weblog-metadata.md | 4 + tests/test_the_test/test_ci_orchestrator.py | 17 ++-- utils/_context/weblog_metadata.py | 14 ---- utils/scripts/build-base-image.py | 41 ---------- .../scripts/ci_orchestrators/workflow_data.py | 22 ++---- utils/scripts/compute-workflow-parameters.py | 9 --- utils/scripts/wait_for_base_image.py | 78 +++++++++++++++++++ 13 files changed, 117 insertions(+), 124 deletions(-) delete mode 100755 utils/scripts/build-base-image.py create mode 100755 utils/scripts/wait_for_base_image.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e50b1d1255..b49270477c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,6 @@ jobs: _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }} _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }} _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }} - _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }} _enable_replay_scenarios: true _system_tests_dev_mode: ${{ matrix.version == 'dev' }} _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }} diff --git a/.github/workflows/compute-workflow-parameters.yml b/.github/workflows/compute-workflow-parameters.yml index f5ac934999a..0405493a000 100644 --- a/.github/workflows/compute-workflow-parameters.yml +++ b/.github/workflows/compute-workflow-parameters.yml @@ -52,11 +52,6 @@ on: default: '' required: false type: string - _build_weblog_base_images: - description: "Shall we build weblog base images" - default: false - required: false - type: boolean # Map the workflow outputs to job outputs outputs: @@ -142,7 +137,6 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ - --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --output $GITHUB_OUTPUT - name: log run: | @@ -156,7 +150,6 @@ jobs: --parametric-job-count ${{ inputs.parametric_job_count }} \ --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \ --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \ - --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \ --format json | jq - name: Extract library target branch diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 56564d9680b..1fc7d6e89f5 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -63,7 +63,7 @@ on: required: false type: boolean _build_weblog_base_image: - description: "Shall we build the weblog base image (only valid if weblog build is local)" + description: "Shall we wait for the weblog base image to be pushed by GitLab CI before building the weblog" default: false required: false type: boolean @@ -146,11 +146,9 @@ jobs: if: inputs._build_lambda_proxy_image run: ./build.sh -i lambda-proxy - - name: Build weblog base image + - name: Wait for weblog base image if: inputs._build_weblog_base_image - run: | - source venv/bin/activate - ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} + run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30 - name: Pull images uses: ./.github/actions/pull_images diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index a783c6f9ca1..9ea0bc6961a 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -44,11 +44,6 @@ on: default: 'custom' required: false type: string - _build_weblog_base_images: - description: "Shall we rebuild weblog base images" - default: false - required: false - type: boolean _build_buddies_images: description: "Shall we build buddies images" default: false @@ -154,7 +149,6 @@ jobs: binaries_artifact: ${{ inputs.binaries_artifact }} _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }} _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }} - _build_weblog_base_images: ${{ inputs._build_weblog_base_images }} parametric: needs: @@ -215,11 +209,9 @@ jobs: run: ls -la binaries/ - name: Export github token to a file run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt" - - name: Build weblog base images - if: matrix.weblog.build_base_image - run: | - source venv/bin/activate - ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} + - name: Wait for weblog base image + if: matrix.weblog.build_weblog_base_image + run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30 - name: Build weblog id: build run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt" diff --git a/docs/CI/github-actions.md b/docs/CI/github-actions.md index 75a2fc61bce..2272e4f7787 100644 --- a/docs/CI/github-actions.md +++ b/docs/CI/github-actions.md @@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI | `_build_buddies_images` | Shall we build buddies images | boolean | false | false | | `_build_proxy_image` | Shall we build proxy image | boolean | false | false | | `_build_lambda_proxy_image` | Shall we build the lambda-proxy image | boolean | false | false | -| `_build_weblog_base_images` | Shall we build weblog base images | boolean | false | false | | `_enable_replay_scenarios` | Enable replay scenarios, should only be used in system-tests CI | boolean | false | false | diff --git a/docs/edit/update-docker-images.md b/docs/edit/update-docker-images.md index a2ae67d46e4..6ebe066acf2 100644 --- a/docs/edit/update-docker-images.md +++ b/docs/edit/update-docker-images.md @@ -1,11 +1,20 @@ Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/). -If you need to update them, you will need to follow those +For weblog base images (nodejs, python, php), the build and push are fully automated: -1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code) -2. create your PR, and add the relevant label to rebuild the image in the CI - * `build-python-base-images` for python weblogs - * `build-php-base-images` for PHP weblogs - * `build-nodejs-base-images` for Node.js weblogs - * `build-proxy-image` for proxy image -3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack) +1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every + branch. For each `docker-bake.hcl` target declared in a library's `weblog_metadata.yml` + `base_image_dependencies` section (see `docs/understand/weblogs/weblog-metadata.md`), it computes a + content hash of the target's dependencies and, if a base image tagged with that hash doesn't already + exist on Docker Hub, builds and pushes it as `-`. +2. If you changed a file listed in `base_image_dependencies`, a new tag will automatically be pushed by + that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to point to the new tag (you can + find it by running `python utils/scripts/build_base_images.py --dry-run` locally). +3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for + the tag referenced in the weblog's `FROM` line to become available before building the weblog, via + `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag + before (or shortly after) your PR's GitHub CI run starts. + +For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a +rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push +your image to hub.docker.com (`#apm-shared-testing` on slack). diff --git a/docs/understand/weblogs/weblog-metadata.md b/docs/understand/weblogs/weblog-metadata.md index 7a121c3131b..2359444cf3b 100644 --- a/docs/understand/weblogs/weblog-metadata.md +++ b/docs/understand/weblogs/weblog-metadata.md @@ -65,3 +65,7 @@ listed paths, then pushes the base image to Docker Hub tagged `- bool: """The run_end_to_end job builds the weblog locally (weblog_build_required).""" return self.build_mode != BuildMode.none - @property - def base_dockerfile(self) -> Path | None: - """Returns the path of the base image docker file if exists, else None""" - image_name = self.base_image_tag - - if image_name is None: - return None - - file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0] - assert file_prefix.endswith(".base") - - path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile") - return path if path.exists() else None - @property def base_image_tag(self) -> str | None: """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" diff --git a/utils/scripts/build-base-image.py b/utils/scripts/build-base-image.py deleted file mode 100755 index f52d9c3ab8b..00000000000 --- a/utils/scripts/build-base-image.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import subprocess -import sys - -from utils._context.weblog_metadata import WeblogMetaData - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build a weblog base image") - parser.add_argument("library", help="Library name (e.g. nodejs, python)") - parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") - args = parser.parse_args() - - weblogs = WeblogMetaData.load(args.library) - names = [w.name for w in weblogs] - - if args.weblog not in names: - print(f"Error: weblog '{args.weblog}' not found for library '{args.library}'") - print(f"Available weblogs: {', '.join(sorted(names))}") - sys.exit(1) - - weblog = next(w for w in weblogs if w.name == args.weblog) - dockerfile = weblog.base_dockerfile - if dockerfile is None: - print(f"Error: no base Dockerfile found for {args.weblog}") - sys.exit(1) - - image_tag = weblog.base_image_tag - if image_tag is None: - print(f"Error: could not extract base image tag for {args.weblog}") - sys.exit(1) - - print(f"Building base image: {image_tag}") - subprocess.run(["docker", "build", "--progress=plain", "-f", str(dockerfile), "-t", image_tag, "."], check=True) - - print(f"\nDone. To push:\n docker push {image_tag}") - - -if __name__ == "__main__": - main() diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 4cdc6100b4e..ba5cb662714 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -206,8 +206,6 @@ def __init__( weblog_instance: int, scenarios_times: dict[str, float], build_time: float, - *, - build_base_images: bool, ): self.library = library self.weblog = weblog @@ -224,9 +222,6 @@ def __init__( # split mechanism self.build_time = build_time - self.build_base_images = build_base_images - """ Shall the end-to-end scenario rebuild the weblog base image for fully baked weblog """ - def serialize(self) -> dict: return { "runs_on": "ubuntu-latest", @@ -237,9 +232,10 @@ def serialize(self) -> dict: "scenarios": sorted(self.scenarios), "expected_job_time": self.expected_job_time + self.build_time, "binaries_artifact": self.weblog.artifact_name, + # only local weblogs build their base image inline; prebuild weblogs wait for it + # once in the dedicated build_end_to_end job (see _get_weblog_build_job). "build_weblog_base_image": self.weblog.build_mode == BuildMode.local - and self.build_base_images - and self.weblog.base_dockerfile is not None, + and self.weblog.base_image_tag is not None, } @property @@ -275,7 +271,6 @@ def split_for_parallel_execution(self, desired_execution_time: float) -> list["J weblog_instance=i + 1, scenarios_times={scenario: self._scenarios_times[scenario] for scenario in scenarios}, build_time=self.build_time, - build_base_images=self.build_base_images, ) ) @@ -314,8 +309,6 @@ def get_endtoend_definitions( maximum_parallel_jobs: int, unique_id: str, binaries_artifact: str, - *, - build_base_images: bool = False, ) -> dict: scenarios = scenario_map.get("endtoend", []) @@ -353,7 +346,6 @@ def get_endtoend_definitions( weblog_instance=1, scenarios_times=scenarios_times, build_time=_get_build_time(library, weblog, time_stats["build"]), - build_base_images=build_base_images, ) ) @@ -372,20 +364,18 @@ def get_endtoend_definitions( "endtoend_defs": { "parallel_enable": len(jobs) > 0, "parallel_weblogs": [ - _get_weblog_build_job(weblog, build_base_images=build_base_images) - for weblog in weblogs - if weblog.build_mode == BuildMode.prebuild + _get_weblog_build_job(weblog) for weblog in weblogs if weblog.build_mode == BuildMode.prebuild ], "parallel_jobs": [job.serialize() for job in jobs], } } -def _get_weblog_build_job(weblog: Weblog, *, build_base_images: bool) -> dict: +def _get_weblog_build_job(weblog: Weblog) -> dict: return { "name": weblog.name, "artifact_name": weblog.artifact_name, - "build_base_images": build_base_images and weblog.base_dockerfile is not None, + "build_weblog_base_image": weblog.base_image_tag is not None, } diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index c004f0621db..07836492c02 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,7 +43,6 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, - build_weblog_base_images: bool = False, ): # this data struture is a dict where: # the key is the workflow identifier @@ -88,7 +87,6 @@ def __init__( maximum_parallel_jobs=256, unique_id=self.unique_id, binaries_artifact=self.binaries_artifact, - build_base_images=build_weblog_base_images, ) self.data["parametric"] = { @@ -288,12 +286,6 @@ def _get_workflow_map( "--system-tests-dev-mode", type=str, help="true if running in system-tests CI, with the dev mode", default="" ) parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) - parser.add_argument( - "--build-weblog-base-images", - type=str, - help="Rebuild weblog base images", - default="", - ) args = parser.parse_args() @@ -313,5 +305,4 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, - build_weblog_base_images=args.build_weblog_base_images == "true", ).export(export_format=args.format, output=args.output) diff --git a/utils/scripts/wait_for_base_image.py b/utils/scripts/wait_for_base_image.py new file mode 100755 index 00000000000..6aeb658451e --- /dev/null +++ b/utils/scripts/wait_for_base_image.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Wait for a weblog's base image to be available on Docker Hub. + +Weblog base images (e.g. `datadog/system-tests:express4.base-`) are built and +pushed by a dedicated GitLab CI job (see `utils/scripts/build_base_images.py`), not by +GitHub Actions. There is no direct dependency mechanism between the two CI systems, so +this script simply polls `docker manifest inspect` for the tag declared in the weblog's +Dockerfile until it appears, or a timeout is reached. + +It never builds or pushes anything: updating the tag in the weblog's Dockerfile after a +base image dependency changes remains the contributor's responsibility (see +docs/edit/update-docker-images.md). + +Deliberately depends only on the standard library (no `utils` package, no venv) so it +can run as a plain CI step before the runner virtualenv is built. +""" + +import argparse +import subprocess +import sys +import time +from pathlib import Path + + +def _base_image_tag(library: str, weblog: str) -> str | None: + """system-tests base image tag read from the first FROM in the weblog Dockerfile.""" + dockerfile = Path(f"utils/build/docker/{library}/{weblog}.Dockerfile") + if not dockerfile.exists(): + print(f"Error: no Dockerfile found for weblog '{weblog}' in library '{library}'") + sys.exit(1) + + for line in dockerfile.read_text().splitlines(): + if line.startswith("FROM "): + image_name = line.split()[1] + return image_name if image_name.startswith("datadog/system-tests:") else None + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description="Wait for a weblog's base image to exist on Docker Hub") + parser.add_argument("library", help="Library name (e.g. nodejs, python)") + parser.add_argument("weblog", help="Weblog name (e.g. express4, flask-poc)") + parser.add_argument("--timeout", type=int, default=900, help="Max time to wait, in seconds (default: 900)") + parser.add_argument("--poll-interval", type=int, default=30, help="Time between polls, in seconds (default: 30)") + args = parser.parse_args() + + image_tag = _base_image_tag(args.library, args.weblog) + + if image_tag is None: + print(f"{args.weblog} does not use a system-tests base image, nothing to wait for") + return + + print(f"Waiting for {image_tag} to be available on Docker Hub (timeout: {args.timeout}s)") + + deadline = time.monotonic() + args.timeout + while True: + result = subprocess.run( + ["docker", "manifest", "inspect", image_tag], + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + print(f"{image_tag} is available") + return + + if time.monotonic() >= deadline: + print(f"Error: timed out waiting for {image_tag}") + if result.stderr: + print(result.stderr.strip()) + sys.exit(1) + + print(f"{image_tag} not found yet, retrying in {args.poll_interval}s...") + time.sleep(args.poll_interval) + + +if __name__ == "__main__": + main()