feat: replace GitHub weblog base image build with a poll for GitLab's push

GitHub Actions has no dependency mechanism on GitLab CI jobs, so instead of
building weblog base images itself (via the removed, already-broken
utils/script/build_base_image.py and PR labels), it now waits for the tag
declared in the weblog's Dockerfile to be pushed by GitLab's build_base_images
job, polling Docker Hub every 30s with a 15 minute timeout
(utils/scripts/wait_for_base_image.py).

Drops the _build_weblog_base_images label-gated flag entirely: waiting is
cheap (a single manifest inspect when the tag already exists) so it now runs
unconditionally for every weblog that has a base image tag.

Author: nccatoni

.github/workflows/ci.yml

@@ -86,7 +86,6 @@ jobs:
       _build_buddies_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-buddies-images') }}
       _build_lambda_proxy_image: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.rebuild_lambda_proxy) }}
       _build_proxy_image: ${{ contains(github.event.pull_request.labels.*.name, 'build-proxy-image') }}
-      _build_weblog_base_images: ${{ contains(github.event.pull_request.labels.*.name, 'build-python-base-images') && matrix.library == 'python' || contains(github.event.pull_request.labels.*.name, 'build-php-base-images') && matrix.library == 'php' || contains(github.event.pull_request.labels.*.name, 'build-nodejs-base-images') && matrix.library == 'nodejs' }}
       _enable_replay_scenarios: true
       _system_tests_dev_mode: ${{ matrix.version == 'dev' }}
       _system_tests_library_target_branch_map: ${{ needs.compute_libraries_and_scenarios.outputs.target-branch-map }}

.github/workflows/compute-workflow-parameters.yml

@@ -52,11 +52,6 @@ on:
         default: ''
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we build weblog base images"
-        default: false
-        required: false
-        type: boolean
 
     # Map the workflow outputs to job outputs
     outputs:
@@ -142,7 +137,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --output $GITHUB_OUTPUT
     - name: log
       run: |
@@ -156,7 +150,6 @@ jobs:
           --parametric-job-count ${{ inputs.parametric_job_count }} \
           --explicit-binaries-artifact "${{ inputs.binaries_artifact }}" \
           --system-tests-dev-mode "${{ inputs._system_tests_dev_mode }}" \
-          --build-weblog-base-images "${{ inputs._build_weblog_base_images }}" \
           --format json | jq
 
     - name: Extract library target branch

.github/workflows/run-end-to-end.yml

@@ -63,7 +63,7 @@ on:
         required: false
         type: boolean
       _build_weblog_base_image:
-        description: "Shall we build the weblog base image (only valid if weblog build is local)"
+        description: "Shall we wait for the weblog base image to be pushed by GitLab CI before building the weblog"
         default: false
         required: false
         type: boolean
@@ -146,11 +146,9 @@ jobs:
       if: inputs._build_lambda_proxy_image
       run: ./build.sh -i lambda-proxy
 
-    - name: Build weblog base image
+    - name: Wait for weblog base image
       if: inputs._build_weblog_base_image
-      run: |
-        source venv/bin/activate
-        ./utils/script/build_base_image.py ${{ inputs.library }} ${{ inputs.weblog }}
+      run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ inputs.weblog }} --timeout 900 --poll-interval 30
 
     - name: Pull images
       uses: ./.github/actions/pull_images

.github/workflows/system-tests.yml

@@ -44,11 +44,6 @@ on:
         default: 'custom'
         required: false
         type: string
-      _build_weblog_base_images:
-        description: "Shall we rebuild weblog base images"
-        default: false
-        required: false
-        type: boolean
       _build_buddies_images:
         description: "Shall we build buddies images"
         default: false
@@ -154,7 +149,6 @@ jobs:
       binaries_artifact: ${{ inputs.binaries_artifact }}
       _system_tests_dev_mode: ${{ inputs._system_tests_dev_mode }}
       _system_tests_library_target_branch_map: ${{ inputs._system_tests_library_target_branch_map }}
-      _build_weblog_base_images: ${{ inputs._build_weblog_base_images }}
 
   parametric:
     needs:
@@ -215,11 +209,9 @@ jobs:
         run: ls -la binaries/
       - name: Export github token to a file
         run: echo "${{ secrets.GITHUB_TOKEN }}" > "$RUNNER_TEMP/github_token.txt"
-      - name: Build weblog base images
-        if: matrix.weblog.build_base_image
-        run: |
-          source venv/bin/activate
-          ./utils/script/build_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }}
+      - name: Wait for weblog base image
+        if: matrix.weblog.build_weblog_base_image
+        run: python3 ./utils/scripts/wait_for_base_image.py ${{ inputs.library }} ${{ matrix.weblog.name }} --timeout 900 --poll-interval 30
       - name: Build weblog
         id: build
         run: SYSTEM_TEST_BUILD_ATTEMPTS=3 ./build.sh ${{ inputs.library }} -i weblog -w ${{ matrix.weblog.name }} -s --github-token-file "$RUNNER_TEMP/github_token.txt"

docs/CI/github-actions.md

@@ -100,5 +100,4 @@ Those parameters are used only by system-tests own CI
 | `_build_buddies_images`               | Shall we build buddies images                                        | boolean | false    | false      |
 | `_build_proxy_image`                  | Shall we build proxy image                                           | boolean | false    | false      |
 | `_build_lambda_proxy_image`           | Shall we build the lambda-proxy image                                | boolean | false    | false      |
-| `_build_weblog_base_images`           | Shall we build weblog base images                                    | boolean | false    | false      |
 | `_enable_replay_scenarios`            | Enable replay scenarios, should only be used in system-tests CI      | boolean | false    | false      |

docs/edit/update-docker-images.md

@@ -1,11 +1,20 @@
 Some of images used in system-tests are prebuild and used threw [hub.docker.com/datadog/system-tests](https://hub.docker.com/repository/docker/datadog/system-tests/).
 
-If you need to update them, you will need to follow those
+For weblog base images (nodejs, python, php), the build and push are fully automated:
 
-1. update the version in the tag for the image you've just modified (there should be 3 or 4 occurences in the code)
-2. create your PR, and add the relevant label to rebuild the image in the CI
-  * `build-python-base-images` for python weblogs
-  * `build-php-base-images` for PHP weblogs
-  * `build-nodejs-base-images` for Node.js weblogs
-  * `build-proxy-image` for proxy image
-3. just before merging your PR, ping somebody from Reliability & Performance team to push your image to hub.docker.com (`#apm-shared-testing` on slack)
+1. GitLab CI's `build_base_images` job (`utils/scripts/build_base_images.py`) runs on every push, on every
+   branch. For each `docker-bake.hcl` target declared in a library's `weblog_metadata.yml`
+   `base_image_dependencies` section (see `docs/understand/weblogs/weblog-metadata.md`), it computes a
+   content hash of the target's dependencies and, if a base image tagged with that hash doesn't already
+   exist on Docker Hub, builds and pushes it as `<base tag>-<hash>`.
+2. If you changed a file listed in `base_image_dependencies`, a new tag will automatically be pushed by
+   that job. Update the `FROM` line of the relevant weblog Dockerfile(s) to point to the new tag (you can
+   find it by running `python utils/scripts/build_base_images.py --dry-run` locally).
+3. GitHub Actions never builds these images itself: it just waits (polling Docker Hub, with a timeout) for
+   the tag referenced in the weblog's `FROM` line to become available before building the weblog, via
+   `utils/scripts/wait_for_base_image.py`. So make sure the GitLab job has had a chance to push the new tag
+   before (or shortly after) your PR's GitHub CI run starts.
+
+For other prebuilt images (e.g. the proxy image), add the `build-proxy-image` label to your PR to force a
+rebuild in GitHub CI; then, just before merging, ping somebody from Reliability & Performance team to push
+your image to hub.docker.com (`#apm-shared-testing` on slack).

docs/understand/weblogs/weblog-metadata.md

@@ -65,3 +65,7 @@ listed paths, then pushes the base image to Docker Hub tagged `<base-tag>-<hash1
 doesn't already exist. It never overwrites an existing tag, so weblog Dockerfiles that `FROM` a
 base image must have their tag updated by hand after a new one is pushed (run the script with
 `--dry-run` to find the current tag for each target).
+
+GitHub Actions never builds these base images itself: `utils/scripts/wait_for_base_image.py`
+polls Docker Hub for the tag currently referenced in the weblog's `FROM` line (with a timeout)
+before building the weblog, since GitLab CI is the only pipeline that builds and pushes them.

tests/test_the_test/test_ci_orchestrator.py

@@ -81,32 +81,27 @@ def test_weblog_build_mode_is_resolved_from_metadata():
 @scenarios.test_the_test
 def test_nodejs_build_base_image():
     scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]}
-    defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True)
+    defs = get_endtoend_definitions("nodejs", scenario_map, [], "dev", 200000, 256, "123", "")
 
     assert defs["endtoend_defs"]["parallel_weblogs"] == []
 
     jobs = {job["weblog"]: job for job in defs["endtoend_defs"]["parallel_jobs"]}
 
-    # express4 is build_mode=local and has a base Dockerfile → should build base image
+    # express4 has a base image tag → should wait for it
     assert jobs["express4"]["build_weblog_base_image"] is True
 
-    # openai-js is build_mode=none and has no base Dockerfile → should not build base image
+    # openai-js has no base image tag → should not wait for a base image
     assert jobs["openai-js@6.0.0"]["build_weblog_base_image"] is False
 
 
 @scenarios.test_the_test
 def test_python_build_base_image():
     scenario_map = {"endtoend": ["DEFAULT", "INTEGRATION_FRAMEWORKS"]}
-    defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "", build_base_images=True)
-
-    # all python weblog has build_mode=prebuild. build_weblog_base_image
-    # only applies to build_mode=local weblogs → should not build base image inline
-    for job in defs["endtoend_defs"]["parallel_jobs"]:
-        assert job["build_weblog_base_image"] is False, job
+    defs = get_endtoend_definitions("python", scenario_map, [], "dev", 200000, 256, "123", "")
 
-    # all python weblog with build_mode=prebuild should rebuild base images in the build job
+    # all python weblogs with a base image tag should wait for it in the build job
     for job in defs["endtoend_defs"]["parallel_weblogs"]:
-        assert job["build_base_images"] is True, job
+        assert job["build_weblog_base_image"] is True, job
 
 
 @scenarios.test_the_test

utils/_context/weblog_metadata.py

@@ -33,20 +33,6 @@ def require_build(self) -> bool:
         """The run_end_to_end job builds the weblog locally (weblog_build_required)."""
         return self.build_mode != BuildMode.none
 
-    @property
-    def base_dockerfile(self) -> Path | None:
-        """Returns the path of the base image docker file if exists, else None"""
-        image_name = self.base_image_tag
-
-        if image_name is None:
-            return None
-
-        file_prefix = image_name.replace("datadog/system-tests:", "").rsplit("-", 1)[0]
-        assert file_prefix.endswith(".base")
-
-        path = Path(f"utils/build/docker/{self.library}/{file_prefix}.Dockerfile")
-        return path if path.exists() else None
-
     @property
     def base_image_tag(self) -> str | None:
         """system-tests base image tag read from the first FROM in the weblog Dockerfile."""