Base image build process

.github/workflows/ci.yml

@@ -86,7 +86,6 @@ jobs:
       _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }}
       _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
-      _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }}
       _enable_replay_scenarios: true
       _system_tests_dev_mode: ${{ matrix.version == 'dev' }}
       _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }}

.github/workflows/compute-workflow-parameters.yml

@@ -52,11 +52,6 @@ on:
         default: ''
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we build weblog base images"
-        default: false
-        required: false
-        type: boolean
 
     # Map the workflow outputs to job outputs
     outputs:
@@ -142,7 +137,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --output $GITHUB_OUTPUT
     - name: log
       run: |
@@ -156,7 +150,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --format json | jq
 
     - name: Extract library target branch

.github/workflows/run-end-to-end.yml

@@ -63,7 +63,7 @@ on:
         required: false
         type: boolean
       _build_weblog_base_image:
-        description: "Shall we build the weblog base image (only valid if weblog build is local)"
+        description: "Shall we wait for the weblog base image to be pushed by GitLab CI before building the weblog"
         default: false
         required: false
         type: boolean
@@ -146,11 +146,9 @@ jobs:
       if: inputs._build_lambda_proxy_image
       run: ./build.sh -i lambda-proxy
 
-    - name: Build weblog base image
+    - name: Wait for weblog base image
       if: inputs._build_weblog_base_image
-      run: |
-        source venv/bin/activate
-        ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }}
+      run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30
 
     - name: Pull images
       uses: ./.github/actions/pull_images

.github/workflows/system-tests.yml

@@ -44,11 +44,6 @@ on:
         default: 'custom'
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we rebuild weblog base images"
-        default: false
-        required: false
-        type: boolean
       _build_buddies_images:
         description: "Shall we build buddies images"
         default: false
@@ -154,7 +149,6 @@ jobs:
       binaries_artifact: ${{ inputs.binaries_artifact }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}
       _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }}
-      _build_weblog_base_images: ${{ inputs._build_weblog_base_images }}
 
   parametric:
     needs:
@@ -215,11 +209,9 @@ jobs:
         run: ls -la binaries/
       - name: Export github token to a file
         run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt"
-      - name: Build weblog base images
-        if: matrix.weblog.build_base_image
-        run: |
-          source venv/bin/activate
-          ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }}
+      - name: Wait for weblog base image
+        if: matrix.weblog.build_weblog_base_image
+        run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30
       - name: Build weblog
         id: build
         run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt"

.gitlab-ci.yml

@@ -474,3 +474,35 @@ mirror_images:
     - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror
   rules:
     - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null'
+
+# ──────────────────────────────────────────────
+# Rebuild weblog base images whose content changed.
+#
+# For every library with a `base_image_dependencies` section in
+# utils/build/docker/<library>/weblog_metadata.yml, computes a content-hash tag per
+# docker-bake.hcl target and pushes it if missing (see utils/scripts/build_base_images.py).
+# Runs on every push, on every branch: it is idempotent (only pushes new tags, never
+# overwrites existing ones), so it is safe to build ahead of merge. Since new tags never
+# replace old ones, a weblog Dockerfile's FROM clause must be updated by hand to pick up
+# a newly pushed base image (run the script with --dry-run to find the new tag).
+# ──────────────────────────────────────────────
+
+build_base_images:
+  image: $CI_IMAGE
+  tags:
+    - docker-in-docker:amd64
+  needs:
+    - job: build_ci_image
+      artifacts: false
+      optional: true
+  stage: e2e
+  before_script:
+    - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text)
+    - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text)
+    - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin
+    - ln -sf /system-tests/venv venv
+    - source venv/bin/activate
+  script:
+    - python utils/scripts/build_base_images.py
+  rules:
+    - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null'

docs/CI/github-actions.md

@@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI
 | `_build_buddies_images`               | Shall we build buddies images                                        | boolean | false    | false      |
 | `_build_proxy_image`                  | Shall we build proxy image                                           | boolean | false    | false      |
 | `_build_lambda_proxy_image`           | Shall we build the lambda-proxy image                                | boolean | false    | false      |
-| `_build_weblog_base_images`           | Shall we build weblog base images                                    | boolean | false    | false      |
 | `_enable_replay_scenarios`            | Enable replay scenarios, should only be used in system-tests CI      | boolean | false    | false      |

docs/edit/update-docker-images.md

@@ -1,11 +1,20 @@
 Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/).
 
-If you need to update them, you will need to follow those
+For weblog base images (nodejs, python, php), the build and push are fully automated:
 
-1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code)
-2. create your PR, and add the relevant label to rebuild the image in the CI
-  * `build-python-base-images` for python weblogs
-  * `build-php-base-images` for PHP weblogs
-  * `build-nodejs-base-images` for Node.js weblogs
-  * `build-proxy-image` for proxy image
-3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack)
+1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every
+   branch. For each `docker-bake.hcl` target declared in a library's `weblog_metadata.yml`
+   `base_image_dependencies` section (see `docs/understand/weblogs/weblog-metadata.md`), it computes a
+   content hash of the target's dependencies and, if a base image tagged with that hash doesn't already
+   exist on Docker Hub, builds and pushes it as `<base tag>-<hash>`.
+2. If you changed a file listed in `base_image_dependencies`, a new tag will automatically be pushed by
+   that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to point to the new tag (you can
+   find it by running `python utils/scripts/build_base_images.py --dry-run` locally).
+3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for
+   the tag referenced in the weblog's `FROM` line to become available before building the weblog, via
+   `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag
+   before (or shortly after) your PR's GitHub CI run starts.
+
+For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a
+rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push
+your image to hub.docker.com (`#apm-shared-testing` on slack).

docs/understand/weblogs/weblog-metadata.md

@@ -45,3 +45,27 @@ produces `openai-js@6.0.0` and `openai-js@7.0.0`.
 `WeblogMetaData.load(library)` in `utils/_context/weblog_metadata.py` merges:
 1. Weblogs discovered from `*.Dockerfile` files in the library folder (default metadata).
 2. Explicit overrides from `weblog_metadata.yml`.
+
+## Base image dependencies
+
+A `weblog_metadata.yml` may also declare a top-level `base_image_dependencies` section, unrelated
+to the per-weblog entries above, used by the `build_base_images` CI job
+(`utils/scripts/build_base_images.py`) to know when a weblog base image needs to be rebuilt:
+
+```yaml
+base_image_dependencies:
+  <docker-bake.hcl target name>:
+    - <path to a file or directory the base image depends on>
+    - ...
+```
+
+For each target listed there, the job computes a content hash from the resolved
+`docker-bake.hcl` target config, the target's Dockerfile, and every git-tracked file under the
+listed paths, then pushes the base image to Docker Hub tagged `<base-tag>-<hash12>` if that tag
+doesn't already exist. It never overwrites an existing tag, so weblog Dockerfiles that `FROM` a
+base image must have their tag updated by hand after a new one is pushed (run the script with
+`--dry-run` to find the current tag for each target).
+
+GitHub Actions never builds these base images itself: `utils/scripts/wait_for_base_image.py`
+polls Docker Hub for the tag currently referenced in the weblog's `FROM` line (with a timeout)
+before building the weblog, since GitLab CI is the only pipeline that builds and pushes them.

tests/test_the_test/test_ci_orchestrator.py

@@ -81,32 +81,27 @@ def test_weblog_build_mode_is_resolved_from_metadata():
 @scenarios.test_the_test
 def test_nodejs_build_base_image():
     scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]}
-    defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True)
+    defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "")
 
     assert defs["endtoend_defs"]["parallel_weblogs"] == []
 
     jobs = {job["weblog"]: job for job in defs["endtoend_defs"]["parallel_jobs"]}
 
-    # express4 is build_mode=local and has a base Dockerfile → should build base image
+    # express4 has a base image tag → should wait for it
     assert jobs["express4"]["build_weblog_base_image"] is True
 
-    # openai-js is build_mode=none and has no base Dockerfile → should not build base image
+    # openai-js has no base image tag → should not wait for a base image
     assert jobs["openai-js@6.0.0"]["build_weblog_base_image"] is False
 
 
 @scenarios.test_the_test
 def test_python_build_base_image():
     scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]}
-    defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True)
-
-    # all python weblog has build_mode=prebuild. build_weblog_base_image
-    # only applies to build_mode=local weblogs → should not build base image inline
-    for job in defs["endtoend_defs"]["parallel_jobs"]:
-        assert job["build_weblog_base_image"] is False, job
+    defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "")
 
-    # all python weblog with build_mode=prebuild should rebuild base images in the build job
+    # all python weblogs with a base image tag should wait for it in the build job
     for job in defs["endtoend_defs"]["parallel_weblogs"]:
-        assert job["build_base_images"] is True, job
+        assert job["build_weblog_base_image"] is True, job
 
 
 @scenarios.test_the_test

utils/_context/weblog_metadata.py

@@ -33,20 +33,6 @@ def require_build(self) -> bool:
         """The run_end_to_end job builds the weblog locally (weblog_build_required)."""
         return self.build_mode != BuildMode.none
 
-    @property
-    def base_dockerfile(self) -> Path | None:
-        """Returns the path of the base image docker file if exists, else None"""
-        image_name = self.base_image_tag
-
-        if image_name is None:
-            return None
-
-        file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0]
-        assert file_prefix.endswith(".base")
-
-        path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile")
-        return path if path.exists() else None
-
     @property
     def base_image_tag(self) -> str | None:
         """system-tests base image tag read from the first FROM in the weblog Dockerfile."""
@@ -68,8 +54,25 @@ def _load_explicit_metadata(library: str) -> dict[str, "WeblogMetaData"]:
         with path.open() as f:
             data: dict = yaml.safe_load(f) or {}
 
+        data.pop("base_image_dependencies", None)
+
         return {name: WeblogMetaData(name=name, library=library, **kwargs) for name, kwargs in data.items()}
 
+    @staticmethod
+    def load_base_image_dependencies(library: str) -> dict[str, list[str]]:
+        """Returns the `base_image_dependencies` section of weblog_metadata.yml: a mapping of
+        docker-bake.hcl target name to the list of paths (files or directories) that base image
+        depends on, used to compute a content-hash tag for the base image build job.
+        """
+        path = Path(f"utils/build/docker/{library}/weblog_metadata.yml")
+        if not path.exists():
+            return {}
+
+        with path.open() as f:
+            data: dict = yaml.safe_load(f) or {}
+
+        return data.get("base_image_dependencies", {}) or {}
+
     @staticmethod
     def load(library: str) -> list["WeblogMetaData"]:
         metadata = WeblogMetaData._load_explicit_metadata(library)

utils/build/docker/nodejs/docker-bake.hcl

@@ -18,33 +18,33 @@ target "express4" {
   inherits   = ["_common"]
   context    = "."
   dockerfile = "utils/build/docker/nodejs/express4.base.Dockerfile"
-  tags       = ["datadog/system-tests:express4.base-v3"]
+  tags       = ["datadog/system-tests:express4.base"]
 }
 
 target "express5" {
   inherits   = ["_common"]
   context    = "."
   dockerfile = "utils/build/docker/nodejs/express5.base.Dockerfile"
-  tags       = ["datadog/system-tests:express5.base-v3"]
+  tags       = ["datadog/system-tests:express5.base"]
 }
 
 target "fastify" {
   inherits   = ["_common"]
   context    = "."
   dockerfile = "utils/build/docker/nodejs/fastify.base.Dockerfile"
-  tags       = ["datadog/system-tests:fastify.base-v3"]
+  tags       = ["datadog/system-tests:fastify.base"]
 }
 
 target "express4-typescript" {
   inherits   = ["_common"]
   context    = "."
   dockerfile = "utils/build/docker/nodejs/express4-typescript.base.Dockerfile"
-  tags       = ["datadog/system-tests:express4-typescript.base-v3"]
+  tags       = ["datadog/system-tests:express4-typescript.base"]
 }
 
 target "nextjs" {
   inherits   = ["_common"]
   context    = "."
   dockerfile = "utils/build/docker/nodejs/nextjs.base.Dockerfile"
-  tags       = ["datadog/system-tests:nextjs.base-v3"]
+  tags       = ["datadog/system-tests:nextjs.base"]
 }

utils/build/docker/nodejs/weblog_metadata.yml

@@ -21,3 +21,26 @@ anthropic-js:
 google_genai-js:
   build_mode: none
   framework_versions: ["1.34.0"]
+
+# Dependencies for base images built by utils/scripts/build_base_images.py, keyed by
+# docker-bake.hcl target name. See docs/understand/weblogs/weblog-metadata.md.
+base_image_dependencies:
+  express4:
+    - utils/build/docker/nodejs/express
+    - utils/build/docker/nodejs/express4/package.json
+    - utils/build/docker/nodejs/express4/bun.lock
+    - utils/build/docker/nodejs/nft-prune.mjs
+  express5:
+    - utils/build/docker/nodejs/express
+    - utils/build/docker/nodejs/express5/package.json
+    - utils/build/docker/nodejs/express5/bun.lock
+    - utils/build/docker/nodejs/nft-prune.mjs
+  express4-typescript:
+    - utils/build/docker/nodejs/express4-typescript
+    - utils/build/docker/nodejs/nft-prune.mjs
+  fastify:
+    - utils/build/docker/nodejs/fastify
+    - utils/build/docker/nodejs/nft-prune.mjs
+  nextjs:
+    - utils/build/docker/nodejs/nextjs
+    - utils/build/docker/nodejs/nft-prune.mjs