Bump Rust to 1.87, Clang/LLVM to 20 (#3926)

* build: update Rust MSRV to 1.87

* ci: add otel exporter for better diagnostics when things go wrong

* ci: limit build parallelism

* bump PHP versions and reduce image size

* Start using bookworm-8 images in CI

* docs: explain minimal rust install

* ci(appsec): adopt bookworm-8

This is going to cause breakages with lints

* ci: use versioned images for shared-ext too

* ci(appsec): clang 17 to clang 20

* ci(appsec): bump Rust to 1.87

* bump PHP and Rust for Windows too

* fix llvm version mismatch and fix cache key

* ci: attempt to build Windows images in CI

* ci: copy bits of windows_git_setup

* ci: windows buildx didn't support --no-cache

* ci: windows didn't even have buildx, way to bury the lede

* ci: avoid docker compose for windows

* appsec: backport fmt lib changes for libddwaf

* ci: verify the compiler shell after install

* ci: windows more verification in tool images

* ci: rename windows build jobs

* add missing docker arg

* wip windows

* upd appsec tag mappings; work around problem due to gradle upgrade

* wip windows

* try to fix vc14

* try to fix vc14 tmp folder

* sleep for a bit before deleting installer to avoid it being locked still

* simplify vc14 install

* adjust vc14 configure args

* oops, full was too much

* revert base images, vc images were replaced

* revert tool images down to just MSRV bump

* patch PHP sdk script

* drop vc14, PHP 7.0/7.1, and fix php-8.0.30.tar.gz hash

* drop the user-agent, it's causing rejections

* move bookworm next to 9

* appsec: bump spdlog's fmt lib too

* style(appsec): format with clang-format 20

* fix(appsec): include order for nonnull

* style(appsec): fix tidy lints, batch 1

* style(appsec): fix tidy lints, batch 2

* fix RemoteConfigTests in new image

* fix TelemetryTests on new images

* drop legacy buster image defintions

* drop legacy centos 6 image defintions

* Fix libdatadog fetch

* ci: restore 7.0/7.1 build images (except for Win)

* xfail Zend/tests/fibers/gh19983.phpt

* ci: delete more PHP 7.0/7.1 Win support

* docker-compose for windows

---------

Co-authored-by: Florian Engelhardt <florian.engelhardt@datadoghq.com>
Co-authored-by: Gustavo André dos Santos Lopes <gustavo.lopes@datadoghq.com>
Co-authored-by: Bob Weinand <bob.weinand@datadoghq.com>

Author: 4 people

.github/workflows/prof_asan.yml

@@ -21,7 +21,7 @@ jobs:
       CARGO_TARGET_DIR: /tmp/build-cargo
       RUST_TOOLCHAIN: nightly-2025-06-13
     container:
-      image: datadog/dd-trace-ci:php-${{matrix.php-version}}_bookworm-7
+      image: datadog/dd-trace-ci:php-${{matrix.php-version}}_bookworm-8
       # https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
       options: --user root
 
@@ -54,10 +54,10 @@ jobs:
           set -eux
           switch-php ${{ matrix.php-build }}
           cd profiling
-          export CC=clang-19
+          export CC=clang-20
           export CFLAGS='-fsanitize=address  -fno-omit-frame-pointer'
           export LDFLAGS='-fsanitize=address -shared-libasan'
-          export RUSTC_LINKER=lld-17
+          export RUSTC_LINKER=lld-20
           triplet=$(uname -m)-unknown-linux-gnu
           RUSTFLAGS='-Zsanitizer=address' cargo +${RUST_TOOLCHAIN} build -Zbuild-std=std,panic_abort --target $triplet --profile profiler-release
           cp -v "$CARGO_TARGET_DIR/$triplet/profiler-release/libdatadog_php_profiling.so" "$(php-config --extension-dir)/datadog-profiling.so"

.github/workflows/prof_correctness.yml

@@ -11,6 +11,8 @@ permissions:
 jobs:
   prof-correctness:
     runs-on: ubuntu-24.04
+    env:
+      LLVM_VERSION: "20"
     strategy:
       fail-fast: false
       matrix:
@@ -46,20 +48,34 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.php-version }}-${{ matrix.phpts }}
+          key: ${{ runner.os }}-cargo-llvm-${{ env.LLVM_VERSION }}-${{ hashFiles('**/Cargo.lock', 'profiling/rust-toolchain.toml') }}-${{ matrix.php-version }}-${{ matrix.phpts }}
 
       - name: Build profiler
         run: |
           codename="$(lsb_release -cs)"
           curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg
-          echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-19 main" | sudo tee /etc/apt/sources.list.d/llvm.list
+          echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${LLVM_VERSION} main" | sudo tee /etc/apt/sources.list.d/llvm.list
           sudo apt-get update
-          sudo apt-get install -y clang-19 lld-19
-          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 100
-          sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-19 100
-          sudo update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-19 100
+          llvm18_packages="$(dpkg-query -W -f='${binary:Package}\n' \
+            '*clang*18*' '*llvm*18*' '*lld*18*' '*libomp*18*' \
+            '*libc++*18*' '*libc++abi*18*' '*mlir*18*' '*flang*18*' \
+            '*bolt*18*' '*polly*18*' 2>/dev/null || true)"
+          if [ -n "$llvm18_packages" ]; then
+            sudo apt-get purge -y $llvm18_packages
+            sudo apt-get autoremove -y
+          fi
+          sudo apt-get install -y clang-${LLVM_VERSION} lld-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev libclang-${LLVM_VERSION}-dev libclang-rt-${LLVM_VERSION}-dev
+          sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 100
+          sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 100
+          sudo update-alternatives --install /usr/bin/ld.lld ld.lld /usr/bin/ld.lld-${LLVM_VERSION} 100
+          export CC=clang-${LLVM_VERSION}
+          export CXX=clang++-${LLVM_VERSION}
+          export LLVM_CONFIG_PATH=/usr/bin/llvm-config-${LLVM_VERSION}
+          export LIBCLANG_PATH=/usr/lib/llvm-${LLVM_VERSION}/lib
+          export LD_LIBRARY_PATH="${LIBCLANG_PATH}:${LD_LIBRARY_PATH:-}"
           clang --version
           ld.lld --version
+          "$LLVM_CONFIG_PATH" --version
           cd profiling
           version_number=$(awk -F' = ' '$1 == "channel" { gsub(/"/, "", $2); print $2 }' rust-toolchain.toml)
           curl https://sh.rustup.rs -sSf | sh -s -- --profile minimal -y --default-toolchain "$version_number"
@@ -74,7 +90,7 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.php-version }}-${{ matrix.phpts }}
+          key: ${{ runner.os }}-cargo-llvm-${{ env.LLVM_VERSION }}-${{ hashFiles('**/Cargo.lock', 'profiling/rust-toolchain.toml') }}-${{ matrix.php-version }}-${{ matrix.phpts }}
 
       - name: Run no profile test
         run: |

.gitlab/build-profiler.sh

@@ -10,16 +10,6 @@ if [ -d '/opt/rh/devtoolset-7' ] ; then
     set -eo pipefail
 fi
 
-# With clang 20, bindgen fails on aarch64:
-#  /usr/lib/llvm20/lib/clang/20/include/arm_vector_types.h:20:9: error: unknown type name '__mfp8'
-#  /usr/lib/llvm20/lib/clang/20/include/arm_vector_types.h:93:24: error: Neon vector size must be 64 or 128 bits
-#  /usr/lib/llvm20/lib/clang/20/include/arm_vector_types.h:94:24: error: Neon vector size must be 64 or 128 bits
-#  /usr/lib/llvm20/lib/clang/20/include/arm_neon.h:6374:25: error: incompatible constant for this __builtin_neon function
-# etc.
-if [ -f /sbin/apk ] && [ $(uname -m) = "aarch64" ]; then
-    ln -sf ../lib/llvm19/bin/clang /usr/bin/clang
-fi
-
 # On CentOS 7 aarch64, clang's resource dir isn't on the default include path,
 # causing bindgen to fail with "stddef.h not found".
 if [ -d '/opt/rh/devtoolset-7' ] && [ "$(uname -m)" = "aarch64" ]; then

.gitlab/ci-images.yml

@@ -17,6 +17,8 @@ CentOS:
   tags: ["arch:amd64"]
   timeout: 4h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
+  variables:
+    DDCI_CONFIGURE_OTEL_EXPORTER: "true"
   parallel:
     matrix:
       - PHP_VERSION:
@@ -46,6 +48,8 @@ Alpine:
   tags: ["arch:amd64"]
   timeout: 4h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
+  variables:
+    DDCI_CONFIGURE_OTEL_EXPORTER: "true"
   parallel:
     matrix:
       - PHP_VERSION:
@@ -75,10 +79,13 @@ Bookworm:
   tags: ["arch:amd64"]
   timeout: 4h
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
+  variables:
+    DDCI_CONFIGURE_OTEL_EXPORTER: "true"
   parallel:
     matrix:
       - PHP_VERSION:
         - base
+      - PHP_VERSION:
         - php-8.5
         - php-8.4
         - php-8.3
@@ -92,38 +99,116 @@ Bookworm:
         - php-7.2
         - php-7.1
         - php-7.0
+        MAKE_JOBS: ["2"]
   script:
     - cd dockerfiles/ci/bookworm
     - echo "$CI_REGISTRY_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
     - docker buildx bake --no-cache --pull --push $PHP_VERSION
 
-Buster:
+.windows_image_build:
   stage: ci-build
   rules:
     - when: manual
       allow_failure: true
   needs: []
-  tags: ["arch:amd64"]
-  timeout: 4h
-  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:29.4.0-noble
+  tags: ["windows-v2:2019"]
+  timeout: 6h
+  variables:
+    DDCI_CONFIGURE_OTEL_EXPORTER: "true"
+    GIT_STRATEGY: none
+  script: |
+    # Kill leftover containers; a previous run may still hold php_ddtrace.dll open.
+    $containers = docker ps -aq 2>$null
+    if ($containers) { docker rm -f $containers 2>$null }
+
+    # Use cmd.exe rd from the parent dir: handles junctions/symlinks that PS5.1 Remove-Item cannot.
+    Write-Host "Performing workspace cleanup..."
+    $workspace = $PWD.Path
+    Push-Location ..
+    cmd /c "rd /s /q ""$workspace"""
+    if (-not (Test-Path $workspace)) {
+        New-Item -ItemType Directory -Path $workspace -Force | Out-Null
+    }
+    Pop-Location
+    $remaining = Get-ChildItem -Path . -Force -ErrorAction SilentlyContinue
+    if ($remaining) { Write-Host "WARNING: could not remove: $($remaining.Name -join ', ')" }
+    Write-Host "Cleanup complete."
+
+    # PS 5.1 ignores $PSNativeCommandUseErrorActionPreference; use $LASTEXITCODE checks instead.
+    $ErrorActionPreference = 'Stop'
+
+    # Manual git clone with proper config.
+    Write-Host "Cloning repository..."
+    git config --global core.longpaths true
+    git config --global core.symlinks true
+    git clone --branch $env:CI_COMMIT_REF_NAME $env:CI_REPOSITORY_URL .
+    if ($LASTEXITCODE -ne 0) {
+        Write-Host "ERROR: git clone failed. Remaining workspace contents:"
+        Get-ChildItem -Force | Select-Object Name
+        exit $LASTEXITCODE
+    }
+    git checkout $env:CI_COMMIT_SHA
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+    # Initialize submodules.
+    Write-Host "Initializing submodules..."
+    git submodule update --init --recursive
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+    Write-Host "Git setup complete."
+
+    # Download docker-compose to the workspace.
+    Write-Host "Downloading docker-compose..."
+    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+    $dockerCompose = "$PWD\docker-compose.exe"
+    Start-BitsTransfer -Source "https://github.com/docker/compose/releases/download/v2.36.0/docker-compose-windows-x86_64.exe" -Destination $dockerCompose
+
+    cd dockerfiles\ci\windows
+
+    $env:DOCKER_CONFIG = Join-Path $env:CI_PROJECT_DIR ".docker"
+    New-Item -ItemType Directory -Force -Path $env:DOCKER_CONFIG | Out-Null
+    Set-Content -Encoding ascii -Path (Join-Path $env:DOCKER_CONFIG "config.json") -Value '{"auths":{}}'
+
+    $env:CI_REGISTRY_TOKEN | docker login -u "$env:CI_REGISTRY_USER" --password-stdin "$env:CI_REGISTRY"
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+    docker version
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+    & $dockerCompose version
+    if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+    foreach ($target in ($env:WINDOWS_IMAGE_TARGETS -split ' ')) {
+      if ([string]::IsNullOrWhiteSpace($target)) { continue }
+
+      Write-Host "Building Windows CI image target $target..."
+      & $dockerCompose build --pull --no-cache $target
+      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+
+      Write-Host "Pushing Windows CI image target $target..."
+      & $dockerCompose push $target
+      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+    }
+
+"Windows 1: Tool Images":
+  extends: .windows_image_build
   parallel:
     matrix:
-      - PHP_VERSION:
-        - base
-        - php-8.5
-        - php-8.4
-        - php-8.3
-        - php-8.2
-        - php-8.1
-        - php-8.0
-        - php-8.0-shared-ext
-        - php-7.4
-        - php-7.4-shared-ext
-        - php-7.3
-        - php-7.2
-        - php-7.1
-        - php-7.0
-  script:
-    - cd dockerfiles/ci/buster
-    - echo "$CI_REGISTRY_TOKEN" | docker login -u "$CI_REGISTRY_USER" --password-stdin "$CI_REGISTRY"
-    - docker buildx bake --no-cache --pull --push $PHP_VERSION
+      - WINDOWS_IMAGE_TARGETS:
+        - "vc15"
+        - "vs16"
+        - "vs17"
+
+"Windows 2: PHP Images":
+  extends: .windows_image_build
+  parallel:
+    matrix:
+      - WINDOWS_IMAGE_TARGETS:
+        - "php-8.5"
+        - "php-8.4"
+        - "php-8.3"
+        - "php-8.2"
+        - "php-8.1"
+        - "php-8.0"
+        - "php-7.4"
+        - "php-7.3"
+        - "php-7.2"