[Backport 7.80.x] [#incident-56663] Prefer DNS endpoint in CI for mirrored Go packages (#52499) (#52698)

Ishirui · ofek · AliDatadog · web-flow · commit 7091f13e8e65 · 2026-06-25T12:08:29.000Z
Backport of #52499 Co-authored-by: ofek <ofekmeister@gmail.com> Co-authored-by: ali.benabdallah <ali.benabdallah@datadoghq.com>
diff --git a/.adms/bazel/adms.mirror.cfg b/.adms/bazel/adms.mirror.cfg
@@ -9,6 +9,9 @@ allow depot-read-api-bzl.us1.ddbuild.io
 allow depot-read-api-cli.us1.ddbuild.io
 allow depot-read-api-generic.us1.ddbuild.io
 allow depot-read-api-go.us1.ddbuild.io
+# Preferred first in GOPROXY (see .adms/go/gitlab.yaml); allow it here so Bazel's
+# downloader doesn't block it.
+allow depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog
 allow depot-read-api-java.us1.ddbuild.io
 allow depot-read-api-python.us1.ddbuild.io
 allow registry.ddbuild.io
diff --git a/.adms/go/gitlab.yaml b/.adms/go/gitlab.yaml
@@ -1,6 +1,6 @@
 # DO NOT EDIT
 
 variables:
-    GOPROXY: "https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/"
+    GOPROXY: "https://depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog:8443/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/"
     GONOSUMDB: "github.com/DataDog,go.ddbuild.io"
     GOPRIVATE: ""
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,5 +1,4 @@
 ---
-
 # Condition mixins for simplification of rules, needed in the include rules so must be defined first
 .if_run_all_e2e_tests: &if_run_all_e2e_tests
   if: $RUN_E2E_TESTS == "on"
@@ -20,8 +19,7 @@
   if: $DEPLOY_AGENT == "true" || $DDR_WORKFLOW_ID != null
 
 # Windows-specific Go source files, pkg/, and cmd/ (27 paths), need to be splitted because Gitlab has a limit to 50 paths in a single rule
-.windows_path: &windows_path
-  # Windows-specific Go source files (by filename convention)
+.windows_path: &windows_path # Windows-specific Go source files (by filename convention)
   - "**/*_windows.go"
 
   # Windows-specific packages in pkg/
@@ -62,8 +60,7 @@
   - bazel/**/*
 
 # Windows-specific comp/, tools, omnibus, containers, tasks, and cross-cutting paths
-.windows_path_2: &windows_path_2
-  # Windows-specific comp/ components
+.windows_path_2: &windows_path_2 # Windows-specific comp/ components
   - comp/systray/**/*
   - comp/updater/**/*
   - comp/checks/agentcrashdetect/**/*
@@ -268,21 +265,21 @@ variables:
   # Build images versions
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to
-  CI_IMAGE_BTF_GEN: v119060732-b96df210
+  CI_IMAGE_BTF_GEN: v120692793-bcaaf163
   CI_IMAGE_BTF_GEN_SUFFIX: ""
-  CI_IMAGE_DOCKER_X64: v119060732-b96df210
+  CI_IMAGE_DOCKER_X64: v120692793-bcaaf163
   CI_IMAGE_DOCKER_X64_SUFFIX: ""
-  CI_IMAGE_DOCKER_ARM64: v119060732-b96df210
+  CI_IMAGE_DOCKER_ARM64: v120692793-bcaaf163
   CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
-  CI_IMAGE_GITLAB_AGENT_DEPLOY: v119060732-b96df210
+  CI_IMAGE_GITLAB_AGENT_DEPLOY: v120692793-bcaaf163
   CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: ""
-  CI_IMAGE_LINUX: v119060732-b96df210
+  CI_IMAGE_LINUX: v120692793-bcaaf163
   CI_IMAGE_LINUX_SUFFIX: ""
-  CI_IMAGE_RPM_X64: v119060732-b96df210
+  CI_IMAGE_RPM_X64: v120692793-bcaaf163
   CI_IMAGE_RPM_X64_SUFFIX: ""
-  CI_IMAGE_RPM_ARM64: v119060732-b96df210
+  CI_IMAGE_RPM_ARM64: v120692793-bcaaf163
   CI_IMAGE_RPM_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_ARMHF: v119060732-b96df210
+  CI_IMAGE_RPM_ARMHF: v120692793-bcaaf163
   CI_IMAGE_RPM_ARMHF_SUFFIX: ""
   CI_IMAGE_WIN_LTSC2022_X64: v119060732-b96df210
   CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""
@@ -420,7 +417,6 @@ variables:
 .if_installer_tests: &if_installer_tests
   if: ($CI_COMMIT_BRANCH == "main"  || $DEPLOY_AGENT == "true" || $RUN_E2E_TESTS == "on" || $DDR_WORKFLOW_ID != null) && $RUN_E2E_TESTS != "off"
 
-
 # When RUN_E2E_TESTS is set to "auto". We do not enforce a behavior for the tests.
 # The behavior of each test will be defined by its rules.
 # For example for new-e2e tests created by each team, here is an example of such rules: https://github.com/DataDog/datadog-agent/blob/ba7079d92077ab5898378594dcafb9cd88a77e57/.gitlab-ci.yml#L1160-L1167
@@ -693,7 +689,6 @@ workflow:
     when: manual
     allow_failure: true
 
-
 .on_main_or_release_branch_or_deploy_always:
   - <<: *if_deploy
     when: always
diff --git a/.gitlab/.pre/common/macos.yml b/.gitlab/.pre/common/macos.yml
@@ -84,6 +84,8 @@
   before_script:
     - !reference [.vault_login]
     - !reference [.aws_retry_config]
+    # Drop GOPROXY entries not usable from macOS runners, before any go command.
+    - !reference [.sanitize_goproxy]
     # Selecting the current Go version
     - |
       eval $(gimme $(cat .go-version))
diff --git a/.gitlab/.pre/deps_fetch/deps_fetch.yml b/.gitlab/.pre/deps_fetch/deps_fetch.yml
@@ -3,7 +3,16 @@
 # Contains jobs which fetch go dependencies, in order
 # to reuse them in further jobs that need them.
 
+# Keep GOPROXY entries usable from the current environment; drop the rest.
+.sanitize_goproxy:
+  - . tools/ci/sanitize-goproxy.sh
+
+# Windows host equivalent, for jobs that forward GOPROXY into a container.
+.sanitize_goproxy_windows:
+  - . ./tools/ci/sanitize-goproxy.ps1
+
 .retrieve_linux_go_deps:
+  - !reference [.sanitize_goproxy]
   - mkdir -p $GOPATH/pkg/mod/cache && zstd -dc modcache.tar.zst | tar xf - -C $GOPATH/pkg/mod/cache
   - rm -f modcache.tar.zst
 
diff --git a/.gitlab/build/bazel/defs.yml b/.gitlab/build/bazel/defs.yml
@@ -85,10 +85,16 @@
 .bazel:runner:macos-amd64:
   extends: .bazel:defs:cache:macos
   tags: [ "macos:sonoma-amd64", "specific:true" ]
+  # Drop GOPROXY entries not usable from macOS runners before bazel forwards
+  # GOPROXY into its repository rules.
+  before_script:
+    - !reference [.sanitize_goproxy]
 
 .bazel:runner:macos-arm64:
   extends: .bazel:defs:cache:macos
   tags: [ "macos:sonoma-arm64", "specific:true" ]
+  before_script:
+    - !reference [.sanitize_goproxy]
 
 .bazel:runner:windows-amd64:
   extends: [ .bazel:defs:cache:windows, .windows_docker_default ]
diff --git a/.gitlab/build/binary_build/schema_generation.yml b/.gitlab/build/binary_build/schema_generation.yml
@@ -83,6 +83,7 @@ generate_config_schema-windows:
     ARCH: "x64"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - >
       .\tools\ci\docker-run-with-bazel-cache.ps1
       -m 8192M
diff --git a/.gitlab/build/source_test/kmt_tasks.yml b/.gitlab/build/source_test/kmt_tasks.yml
@@ -40,6 +40,8 @@ test_kmt_local_setup_macos:
   before_script:
     - !reference [.vault_login]
     - !reference [.aws_retry_config]
+    # Drop GOPROXY entries not usable from macOS runners, before any go command.
+    - !reference [.sanitize_goproxy]
     - !reference [.setup_github_token_read]
     # Selecting the current Go version
     - |
diff --git a/.gitlab/windows/build/binary_build/windows.yml b/.gitlab/windows/build/binary_build/windows.yml
@@ -11,6 +11,7 @@ build_windows_container_entrypoint:
     ARCH: "x64"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - if (Test-Path build-out) { remove-item -recurse -force build-out }
     - >
       docker run --rm
diff --git a/.gitlab/windows/build/lint/windows.yml b/.gitlab/windows/build/lint/windows.yml
@@ -6,6 +6,7 @@
   extends: .windows_docker_default
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - !reference [.docker_pull_winbuildimage_instrumented]
     # Previously this job required only 8Gb of memory but since Go 1.20 it requires more to avoid being OOM killed.
     # Each Windows VM has 32Gb of memory and contains 3 runners that can run one job at a time each (so a maximum of 3 simultaneous jobs per VM).
diff --git a/.gitlab/windows/build/package_build/windows.yml b/.gitlab/windows/build/package_build/windows.yml
@@ -5,6 +5,7 @@
   needs: ["go_deps"]
   script:
     - $ErrorActionPreference = 'Stop'
+    - !reference [.sanitize_goproxy_windows]
     - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg }
     - mkdir omnibus\pkg
     - >
@@ -97,6 +98,7 @@ windows_msi_and_bosh_zip_x64-a7-fips:
     ARCH: "x64"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg }
     - mkdir omnibus\pkg
     - >
diff --git a/.gitlab/windows/build/source_test/windows.yml b/.gitlab/windows/build/source_test/windows.yml
@@ -11,6 +11,7 @@
     - .bazel:defs:cache:windows
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags
     # inside JUNIT_TAR and then later used by datadog-ci
     - !reference [.docker_pull_winbuildimage_instrumented]
@@ -80,6 +81,7 @@ tests_windows-x64:
   extends: [.windows_docker_default, .bazel:defs:cache:windows]
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - >
       .\tools\ci\docker-run-with-bazel-cache.ps1
       -m 16384M
@@ -108,6 +110,7 @@ tests_windows-x64:
   extends: [.windows_docker_default, .bazel:defs:cache:windows]
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - >
       .\tools\ci\docker-run-with-bazel-cache.ps1
       -m 16384M
diff --git a/.gitlab/windows/test/integration_test/windows.yml b/.gitlab/windows/test/integration_test/windows.yml
@@ -14,6 +14,7 @@
     - Remove-Item "$tmpfile"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags
     # inside JUNIT_TAR and then later used by datadog-ci
     - >
diff --git a/Dockerfiles/agent-ddot/Dockerfile.agent-otel b/Dockerfiles/agent-ddot/Dockerfile.agent-otel
@@ -116,21 +116,30 @@ RUN ARCH=$(dpkg --print-architecture) && \
 COPY manifest.yaml /workspace/datadog-agent/comp/otelcol/collector-contrib/impl/manifest.yaml
 
 # Generate the files and clean up go cache to free space
-RUN GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB dda inv collector.generate && \
+ARG GOPROXY
+ARG GONOSUMDB
+# The GOPROXY build arg is re-injected into every RUN below as an env var, so
+# strip any unreachable entry once here and apply the result explicitly to each
+# go step (an explicit assignment overrides the injected arg). Mirrors
+# tools/ci/sanitize-goproxy.sh.
+RUN printf '%s' "$GOPROXY" | tr '|' '\n' \
+      | grep -vE '^https?://[^/]*\.fabric\.dog(:[0-9]+)?(/|$)' \
+      | paste -sd '|' - > /tmp/goproxy.sanitized
+RUN GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB dda inv collector.generate && \
     go clean -cache -modcache
 
 # Build the OTel agent with cleanup
 RUN if [ "$PACKAGE_TYPE" = "windows" ]; then \
-    GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CXX=x86_64-w64-mingw32-g++ CC=x86_64-w64-mingw32-gcc dda inv otel-agent.build --byoc && \
+    GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CXX=x86_64-w64-mingw32-g++ CC=x86_64-w64-mingw32-gcc dda inv otel-agent.build --byoc && \
     go clean -cache; \
     else \
-    GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB dda inv otel-agent.build --byoc && \
+    GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB dda inv otel-agent.build --byoc && \
     go clean -cache; \
     fi
 
 # Build the build-ddot-byoc tool (required for linux and windows PACKAGE_TYPE)
 RUN if [ "$PACKAGE_TYPE" = "linux" ] || [ "$PACKAGE_TYPE" = "windows" ]; then \
-    cd /workspace/datadog-agent/tools/build-ddot-byoc && GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB GOWORK=off go build -o /usr/local/bin/build-ddot-byoc .; \
+    cd /workspace/datadog-agent/tools/build-ddot-byoc && GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB GOWORK=off go build -o /usr/local/bin/build-ddot-byoc .; \
     fi
 
 # Build packages if PACKAGE_TYPE is specified, otherwise just output the binary.
diff --git a/tools/ci/sanitize-goproxy.ps1 b/tools/ci/sanitize-goproxy.ps1
@@ -0,0 +1,24 @@
+# Normalize $env:GOPROXY on Windows: drop the entry that isn't usable here
+# (hosts ending in .fabric.dog), keeping the rest. Mirrors tools/ci/sanitize-goproxy.sh.
+#
+# Dot-source before `docker run ... -e GOPROXY` so the container inherits the
+# cleaned value:  . ./tools/ci/sanitize-goproxy.ps1
+
+$sgpProbeUrl = 'https://depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog:8443/magicmirror/magicmirror/@current/sumdb/sum.golang.org/supported'
+
+function Test-SgpUsable {
+    try {
+        # -SkipHttpErrorCheck (PS7+) so a non-2xx still counts as completed; any
+        # failure throws and is caught below.
+        Invoke-WebRequest -Uri $sgpProbeUrl -Method Head -TimeoutSec 5 -SkipHttpErrorCheck -UseBasicParsing | Out-Null
+        return $true
+    } catch {
+        return $false
+    }
+}
+
+if ($env:GOPROXY -and $env:GOPROXY.Contains('.fabric.dog') -and -not (Test-SgpUsable)) {
+    $env:GOPROXY = (($env:GOPROXY -split '\|') |
+        Where-Object { $_ -notmatch '^https?://[^/]*\.fabric\.dog(:\d+)?(/|$)' }) -join '|'
+    Write-Host "sanitize-goproxy: endpoint unusable here; stripped it from GOPROXY"
+}
diff --git a/tools/ci/sanitize-goproxy.sh b/tools/ci/sanitize-goproxy.sh
@@ -0,0 +1,29 @@
+# shellcheck shell=bash
+# Drop the GOPROXY entry that isn't usable from the current environment (hosts
+# ending in .fabric.dog), keeping the rest. Probes the endpoint the way Go would
+# (an HTTPS request with cert verification) and strips it only if that fails;
+# missing curl counts as unusable.
+#
+# Source this file (don't execute) so the change is visible to the caller:
+#   . tools/ci/sanitize-goproxy.sh
+# Safe to source repeatedly; never empties GOPROXY.
+
+__sgp_probe_url="https://depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog:8443/magicmirror/magicmirror/@current/sumdb/sum.golang.org/supported"
+
+# Returns 0 if the endpoint completes a request over verified TLS. No --fail: we
+# only care that the request completes, not its HTTP status.
+__sgp_usable() {
+    command -v curl >/dev/null 2>&1 \
+        && curl --silent --output /dev/null --connect-timeout 3 --max-time 5 "$__sgp_probe_url"
+}
+
+case "${GOPROXY-}" in
+    *.fabric.dog*)
+        if ! __sgp_usable; then
+            GOPROXY="$(printf '%s' "$GOPROXY" | tr '|' '\n' \
+                | grep -vE '^https?://[^/]*\.fabric\.dog(:[0-9]+)?(/|$)' | paste -sd '|' -)"
+            export GOPROXY
+            echo "sanitize-goproxy: endpoint unusable here; stripped it from GOPROXY" >&2
+        fi
+        ;;
+esac
