Prefer DNS endpoint in CI for mirrored Go packages (#52499)

ofek · web-flow · commit ceea175a0fef · 2026-06-23T08:47:51.000Z
### Motivation Service-to-service traffic is supposed to use internal DNS instead of the currently-prioritized gateway proxy, which intermittently throws 503s even though the underlying service is healthy. ### Additional Notes CI will pass when the build images have been updated, as seen here #52547 Co-authored-by: ofek.lev <ofek.lev@datadoghq.com>
diff --git a/.adms/bazel/adms.mirror.cfg b/.adms/bazel/adms.mirror.cfg
@@ -9,6 +9,9 @@ allow depot-read-api-bzl.us1.ddbuild.io
 allow depot-read-api-cli.us1.ddbuild.io
 allow depot-read-api-generic.us1.ddbuild.io
 allow depot-read-api-go.us1.ddbuild.io
+# Preferred first in GOPROXY (see .adms/go/gitlab.yaml); allow it here so Bazel's
+# downloader doesn't block it.
+allow depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog
 allow depot-read-api-java.us1.ddbuild.io
 allow depot-read-api-python.us1.ddbuild.io
 allow mass-read.us1.ddbuild.io
diff --git a/.adms/go/gitlab.yaml b/.adms/go/gitlab.yaml
@@ -1,6 +1,6 @@
 # DO NOT EDIT
 
 variables:
-    GOPROXY: "https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/"
+    GOPROXY: "https://depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog:8443/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/magicmirror/@current/|https://depot-read-api-go.us1.ddbuild.io/magicmirror/testing/@current/"
     GONOSUMDB: "github.com/DataDog,go.ddbuild.io"
     GOPRIVATE: ""
diff --git a/.gitlab/.pre/common/macos.yml b/.gitlab/.pre/common/macos.yml
@@ -84,6 +84,8 @@
   before_script:
     - !reference [.vault_login]
     - !reference [.aws_retry_config]
+    # Drop GOPROXY entries not usable from macOS runners, before any go command.
+    - !reference [.sanitize_goproxy]
     # Selecting the current Go version
     - |
       eval $(gimme $(cat .go-version))
diff --git a/.gitlab/.pre/deps_fetch/deps_fetch.yml b/.gitlab/.pre/deps_fetch/deps_fetch.yml
@@ -3,7 +3,16 @@
 # Contains jobs which fetch go dependencies, in order
 # to reuse them in further jobs that need them.
 
+# Keep GOPROXY entries usable from the current environment; drop the rest.
+.sanitize_goproxy:
+  - . tools/ci/sanitize-goproxy.sh
+
+# Windows host equivalent, for jobs that forward GOPROXY into a container.
+.sanitize_goproxy_windows:
+  - . ./tools/ci/sanitize-goproxy.ps1
+
 .retrieve_linux_go_deps:
+  - !reference [.sanitize_goproxy]
   - mkdir -p $GOPATH/pkg/mod/cache && zstd -dc modcache.tar.zst | tar xf - -C $GOPATH/pkg/mod/cache
   - rm -f modcache.tar.zst
 
diff --git a/.gitlab/build/bazel/defs.yml b/.gitlab/build/bazel/defs.yml
@@ -85,10 +85,16 @@
 .bazel:runner:macos-amd64:
   extends: .bazel:defs:cache:macos
   tags: [ "macos:sonoma-amd64", "specific:true" ]
+  # Drop GOPROXY entries not usable from macOS runners before bazel forwards
+  # GOPROXY into its repository rules.
+  before_script:
+    - !reference [.sanitize_goproxy]
 
 .bazel:runner:macos-arm64:
   extends: .bazel:defs:cache:macos
   tags: [ "macos:sonoma-arm64", "specific:true" ]
+  before_script:
+    - !reference [.sanitize_goproxy]
 
 .bazel:runner:windows-amd64:
   extends: [ .bazel:defs:cache:windows, .windows_docker_default ]
diff --git a/.gitlab/build/binary_build/schema_generation.yml b/.gitlab/build/binary_build/schema_generation.yml
@@ -81,6 +81,7 @@ generate_config_schema-windows:
     ARCH: "x64"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - >
       .\tools\ci\docker-run-with-bazel-cache.ps1
       -m 24576M
diff --git a/.gitlab/build/source_test/kmt_tasks.yml b/.gitlab/build/source_test/kmt_tasks.yml
@@ -40,6 +40,8 @@ test_kmt_local_setup_macos:
   before_script:
     - !reference [.vault_login]
     - !reference [.aws_retry_config]
+    # Drop GOPROXY entries not usable from macOS runners, before any go command.
+    - !reference [.sanitize_goproxy]
     - !reference [.setup_github_token_read]
     # Selecting the current Go version
     - |
diff --git a/.gitlab/windows/build/binary_build/windows.yml b/.gitlab/windows/build/binary_build/windows.yml
@@ -11,6 +11,7 @@ build_windows_container_entrypoint:
     ARCH: "x64"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - if (Test-Path build-out) { remove-item -recurse -force build-out }
     - >
       docker run --rm
diff --git a/.gitlab/windows/build/lint/windows.yml b/.gitlab/windows/build/lint/windows.yml
@@ -4,6 +4,7 @@
   extends: .windows_docker_default
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - !reference [.docker_pull_winbuildimage_instrumented]
     # Previously this job required only 8Gb of memory but since Go 1.20 it requires more to avoid being OOM killed.
     # Each Windows VM has 32Gb of memory and contains 3 runners that can run one job at a time each (so a maximum of 3 simultaneous jobs per VM).
diff --git a/.gitlab/windows/build/package_build/windows.yml b/.gitlab/windows/build/package_build/windows.yml
@@ -5,6 +5,7 @@
   needs: ["go_deps"]
   script:
     - $ErrorActionPreference = 'Stop'
+    - !reference [.sanitize_goproxy_windows]
     - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg }
     - mkdir omnibus\pkg
     - >
@@ -97,6 +98,7 @@ windows_msi_and_bosh_zip_x64-a7-fips:
     ARCH: "x64"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - if (Test-Path omnibus\pkg) { remove-item -recurse -force omnibus\pkg }
     - mkdir omnibus\pkg
     - >
diff --git a/.gitlab/windows/build/source_test/windows.yml b/.gitlab/windows/build/source_test/windows.yml
@@ -11,6 +11,7 @@
     - .bazel:defs:cache:windows
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags
     # inside JUNIT_TAR and then later used by datadog-ci
     - !reference [.docker_pull_winbuildimage_instrumented]
@@ -81,6 +82,7 @@ tests_windows-x64:
   extends: [.windows_docker_default, .bazel:defs:cache:windows]
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - >
       .\tools\ci\docker-run-with-bazel-cache.ps1
       -m 16384M
@@ -109,6 +111,7 @@ tests_windows-x64:
   extends: [.windows_docker_default, .bazel:defs:cache:windows]
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     - >
       .\tools\ci\docker-run-with-bazel-cache.ps1
       -m 16384M
diff --git a/.gitlab/windows/test/integration_test/windows.yml b/.gitlab/windows/test/integration_test/windows.yml
@@ -14,6 +14,7 @@
     - Remove-Item "$tmpfile"
   script:
     - $ErrorActionPreference = "Stop"
+    - !reference [.sanitize_goproxy_windows]
     # we pass in CI_JOB_URL and CI_JOB_NAME so that they can be added to additional tags
     # inside JUNIT_TAR and then later used by datadog-ci
     - >
diff --git a/Dockerfiles/agent-ddot/Dockerfile.agent-otel b/Dockerfiles/agent-ddot/Dockerfile.agent-otel
@@ -113,21 +113,28 @@ COPY manifest.yaml /workspace/datadog-agent/comp/otelcol/collector-contrib/impl/
 # Generate the files and clean up go cache to free space
 ARG GOPROXY
 ARG GONOSUMDB
-RUN GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB dda inv collector.generate && \
+# The GOPROXY build arg is re-injected into every RUN below as an env var, so
+# strip any unreachable entry once here and apply the result explicitly to each
+# go step (an explicit assignment overrides the injected arg). Mirrors
+# tools/ci/sanitize-goproxy.sh.
+RUN printf '%s' "$GOPROXY" | tr '|' '\n' \
+      | grep -vE '^https?://[^/]*\.fabric\.dog(:[0-9]+)?(/|$)' \
+      | paste -sd '|' - > /tmp/goproxy.sanitized
+RUN GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB dda inv collector.generate && \
     go clean -cache -modcache
 
 # Build the OTel agent with cleanup
 RUN if [ "$PACKAGE_TYPE" = "windows" ]; then \
-    GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CXX=x86_64-w64-mingw32-g++ CC=x86_64-w64-mingw32-gcc dda inv otel-agent.build --byoc && \
+    GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CXX=x86_64-w64-mingw32-g++ CC=x86_64-w64-mingw32-gcc dda inv otel-agent.build --byoc && \
     go clean -cache; \
     else \
-    GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB dda inv otel-agent.build --byoc && \
+    GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB dda inv otel-agent.build --byoc && \
     go clean -cache; \
     fi
 
 # Build the build-ddot-byoc tool (required for linux and windows PACKAGE_TYPE)
 RUN if [ "$PACKAGE_TYPE" = "linux" ] || [ "$PACKAGE_TYPE" = "windows" ]; then \
-    cd /workspace/datadog-agent/tools/build-ddot-byoc && GOPROXY=$GOPROXY GONOSUMDB=$GONOSUMDB GOWORK=off go build -o /usr/local/bin/build-ddot-byoc .; \
+    cd /workspace/datadog-agent/tools/build-ddot-byoc && GOPROXY="$(cat /tmp/goproxy.sanitized)" GONOSUMDB=$GONOSUMDB GOWORK=off go build -o /usr/local/bin/build-ddot-byoc .; \
     fi
 
 # Build packages if PACKAGE_TYPE is specified, otherwise just output the binary.
diff --git a/tools/ci/sanitize-goproxy.ps1 b/tools/ci/sanitize-goproxy.ps1
@@ -0,0 +1,24 @@
+# Normalize $env:GOPROXY on Windows: drop the entry that isn't usable here
+# (hosts ending in .fabric.dog), keeping the rest. Mirrors tools/ci/sanitize-goproxy.sh.
+#
+# Dot-source before `docker run ... -e GOPROXY` so the container inherits the
+# cleaned value:  . ./tools/ci/sanitize-goproxy.ps1
+
+$sgpProbeUrl = 'https://depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog:8443/magicmirror/magicmirror/@current/sumdb/sum.golang.org/supported'
+
+function Test-SgpUsable {
+    try {
+        # -SkipHttpErrorCheck (PS7+) so a non-2xx still counts as completed; any
+        # failure throws and is caught below.
+        Invoke-WebRequest -Uri $sgpProbeUrl -Method Head -TimeoutSec 5 -SkipHttpErrorCheck -UseBasicParsing | Out-Null
+        return $true
+    } catch {
+        return $false
+    }
+}
+
+if ($env:GOPROXY -and $env:GOPROXY.Contains('.fabric.dog') -and -not (Test-SgpUsable)) {
+    $env:GOPROXY = (($env:GOPROXY -split '\|') |
+        Where-Object { $_ -notmatch '^https?://[^/]*\.fabric\.dog(:\d+)?(/|$)' }) -join '|'
+    Write-Host "sanitize-goproxy: endpoint unusable here; stripped it from GOPROXY"
+}
diff --git a/tools/ci/sanitize-goproxy.sh b/tools/ci/sanitize-goproxy.sh
@@ -0,0 +1,29 @@
+# shellcheck shell=bash
+# Drop the GOPROXY entry that isn't usable from the current environment (hosts
+# ending in .fabric.dog), keeping the rest. Probes the endpoint the way Go would
+# (an HTTPS request with cert verification) and strips it only if that fails;
+# missing curl counts as unusable.
+#
+# Source this file (don't execute) so the change is visible to the caller:
+#   . tools/ci/sanitize-goproxy.sh
+# Safe to source repeatedly; never empties GOPROXY.
+
+__sgp_probe_url="https://depot-read-api-go.rapid-dependency-management-depot.all-clusters.local-dc.fabric.dog:8443/magicmirror/magicmirror/@current/sumdb/sum.golang.org/supported"
+
+# Returns 0 if the endpoint completes a request over verified TLS. No --fail: we
+# only care that the request completes, not its HTTP status.
+__sgp_usable() {
+    command -v curl >/dev/null 2>&1 \
+        && curl --silent --output /dev/null --connect-timeout 3 --max-time 5 "$__sgp_probe_url"
+}
+
+case "${GOPROXY-}" in
+    *.fabric.dog*)
+        if ! __sgp_usable; then
+            GOPROXY="$(printf '%s' "$GOPROXY" | tr '|' '\n' \
+                | grep -vE '^https?://[^/]*\.fabric\.dog(:[0-9]+)?(/|$)' | paste -sd '|' -)"
+            export GOPROXY
+            echo "sanitize-goproxy: endpoint unusable here; stripped it from GOPROXY" >&2
+        fi
+        ;;
+esac
