diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d250260a165..25001c89b942 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,7 +184,8 @@ build: -e CI_PROJECT_NAME ` -e CI_JOB_NAME_SLUG ` $env:WINDOWS_BUILD_IMAGE ` - Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi DownloadWinSsiTelemetryForwarder + Info Clean BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet ` + --NugetPackageDirectory c:\mnt\packages $buildExitCode = $LASTEXITCODE Wait-Job $statsJob -Timeout 15 | Out-Null @@ -198,15 +199,104 @@ build: if ($buildExitCode -ne 0) { throw "Docker build container exited with code $buildExitCode" } + - | + # Run the native tests against the outputs from the build above, matching the + # Azure pipeline's build-then-test flow in a shared working directory. + docker run --rm -m 20480M ` + -v "$(Get-Location):c:\mnt" ` + -e CI_JOB_ID=${CI_JOB_ID} ` + -e ENABLE_MULTIPROCESSOR_COMPILATION=true ` + -e CL_MPCount=16 ` + -e WINDOWS_BUILDER=true ` + -e AWS_NETWORKING=true ` + -e NUGET_CERT_REVOCATION_MODE=offline ` + -e DD_LOGGER_DD_API_KEY ` + -e CI_IDENTITIES_GITLAB_ID_TOKEN ` + -e CI_PROJECT_NAME ` + -e CI_JOB_NAME_SLUG ` + $env:WINDOWS_BUILD_IMAGE ` + CompileTracerNativeTests RunTracerNativeTests ` + CompileNativeLoaderNativeTests RunNativeLoaderNativeTests ` + CompileProfilerNativeTests RunProfilerNativeTests ` + --NugetPackageDirectory c:\mnt\packages + if ($LASTEXITCODE -ne 0) { + throw "Native test container exited with code $LASTEXITCODE" + } + - | + # Start the managed Windows unit-test migration with one modern .NET target + # and .NET Framework. Preserve each framework's results before the next NUKE + # invocation cleans the shared results and logs directories. + $managedTestFrameworks = @("net8.0", "net48") + foreach ($framework in $managedTestFrameworks) { + Write-Output "Building and running managed unit tests for $framework" + docker run --rm -m 20480M ` + -v "$(Get-Location):c:\mnt" ` + -e CI_JOB_ID=${CI_JOB_ID} ` + -e WINDOWS_BUILDER=true ` + -e AWS_NETWORKING=true ` + -e NUGET_CERT_REVOCATION_MODE=offline ` + -e DD_LOGGER_DD_API_KEY ` + -e CI_IDENTITIES_GITLAB_ID_TOKEN ` + -e CI_PROJECT_NAME ` + -e CI_JOB_NAME_SLUG ` + $env:WINDOWS_BUILD_IMAGE ` + BuildManagedUnitTests RunManagedUnitTests ` + --framework $framework ` + --NugetPackageDirectory c:\mnt\packages + $managedTestExitCode = $LASTEXITCODE + + $frameworkArtifacts = Join-Path "artifacts/build_data/managed-unit-tests" $framework + New-Item -ItemType Directory -Path $frameworkArtifacts -Force | Out-Null + foreach ($directory in @("results", "logs")) { + $source = Join-Path "artifacts/build_data" $directory + if (Test-Path $source) { + Copy-Item -Path $source -Destination $frameworkArtifacts -Recurse -Force + } + } + + if ($managedTestExitCode -ne 0) { + throw "Managed unit tests for $framework exited with code $managedTestExitCode" + } + } + - | + # Package and sign only after the native tests pass so the release outputs are + # produced from the successfully tested build state. + docker run --rm -m 20480M ` + -v "$(Get-Location):c:\mnt" ` + -e CI_JOB_ID=${CI_JOB_ID} ` + -e ENABLE_MULTIPROCESSOR_COMPILATION=true ` + -e CL_MPCount=16 ` + -e WINDOWS_BUILDER=true ` + -e AWS_NETWORKING=true ` + -e SIGN_WINDOWS=true ` + -e NUGET_CERT_REVOCATION_MODE=offline ` + -e CI_IDENTITIES_GITLAB_ID_TOKEN ` + -e CI_PROJECT_NAME ` + -e CI_JOB_NAME_SLUG ` + $env:WINDOWS_BUILD_IMAGE ` + PublishFleetInstaller PackageTracerHome ZipSymbols SignDlls SignMsi DownloadWinSsiTelemetryForwarder ` + --NugetPackageDirectory c:\mnt\packages + if ($LASTEXITCODE -ne 0) { + throw "Docker packaging container exited with code $LASTEXITCODE" + } - mkdir artifacts-out - xcopy /e/s build-out\${CI_JOB_ID}\*.* artifacts-out - remove-item -recurse -force build-out\${CI_JOB_ID} - get-childitem build-out - get-childitem artifacts-out artifacts: + when: always expire_in: 2 weeks paths: - artifacts-out + - artifacts/build_data/tests + - artifacts/build_data/managed-unit-tests + - artifacts/build_data/dumps + - profiler/build_data/tests + reports: + junit: + - artifacts/build_data/tests/*.xml + - profiler/build_data/tests/*.xml id_tokens: CI_IDENTITIES_GITLAB_ID_TOKEN: aud: ci-identities diff --git a/GitlabMigration.md b/GitlabMigration.md new file mode 100644 index 000000000000..c3f33f800bfa --- /dev/null +++ b/GitlabMigration.md @@ -0,0 +1,751 @@ +# GitLab CI migration plan and findings + +Last updated: 2026-08-03 + +## Purpose + +Migrate `dd-trace-dotnet` CI from Azure DevOps to GitLab without rewriting the NUKE build system. The migration is additive: Azure DevOps remains unchanged and continues to run during a long parallel-validation period. Duplicated work is acceptable until GitLab demonstrates equivalent builds, tests, artifacts, and reliability. + +This document combines: + +- Guidance received from the CI Infrastructure team. +- The proposed phased migration plan. +- Completed Windows GitLab performance and log-quality work. +- Findings from the Windows runner-image and vcpkg investigations. +- The remaining implementation and validation work. + +## Help and documentation + +### Support thread + +- Slack thread: [request for help in `#ci-infa-support`](https://dd.slack.com/archives/C06TGQ49U1Z/p1777285094950679) +- Help from Jorge Torres Martinez on 2026-04-27 at 14:30 UTC. + +### Internal documentation + +- [Windows Runners V2](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/5091983382/Windows+Runners+V2) +- [Create custom macOS Runner](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/3457515540/Create+custom+macOS+Runner) +- [Getting Started with macOS Runners](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/3557065191/Getting+Started+with+macOS+Runners) +- [Onboarding To GitLab CI](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/2411528655/Onboarding+To+Gitlab+CI) +- [GitLab User Documentation](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/2411561449/Gitlab+User+Documentation) +- [GitLab CI/CD YAML reference](https://docs.gitlab.com/ci/yaml/) +- [CI job-sizing dashboard](https://app.datadoghq.com/dashboard/vev-54z-7wr/?fromUser=false&refresh_mode=sliding&from_ts=1776091263874&to_ts=1777300863874&live=true) + +## Current state + +CI is split across Azure DevOps and GitLab. + +- `.azure-pipelines/ultimate-pipeline.yml` contains the full build and test matrix. As of this update, it is 5,693 lines with 80 top-level stages. +- Azure DevOps covers Windows, Linux x64 and ARM64, glibc and musl, macOS, unit tests, integration tests, smoke tests, profiler tests, packaging, and publishing. +- GitLab currently runs the Windows build and native unit tests in one job, followed by packaging and publishing work, plus benchmarks. +- Existing GitLab jobs import Azure artifacts through `.gitlab/download-single-step-artifacts.sh` and `.gitlab/download-serverless-artifacts.sh`. +- The proposed `.gitlab/ci/` Phase 1 files do not exist yet. + +The long-term goal is to host the entire build and test pipeline in GitLab, remove the cross-CI artifact handoff, and retire Azure DevOps only after an extended period of demonstrated parity. + +## Runner guidance + +| Workload | Runner tags | Characteristics and intended use | +| --- | --- | --- | +| Linux without a Docker daemon | `arch:amd64`, `arch:arm64` | Kubernetes pod runners. A job may specify its image, sidecars, CPU, and memory. Docker images can be built through the remote BuildKit executor. | +| Linux requiring Docker | `docker-in-docker:amd64`, `docker-in-docker:arm64` | Kubernetes microVM runners with a Docker daemon. Use for Testcontainers, Docker Compose, `kind`, and similar workloads. | +| Windows | `windows-v2:2019`, `windows-v2:2022`, `windows-v2:2025` | Shared, persistent runners. Jobs should use Docker images to isolate dependencies and state. | +| Shared macOS | `macos:sonoma-arm64`, `macos:sonoma-amd64` | Shared native macOS runners available across repositories. | +| Dedicated macOS | Team-specific | Available when a specialized configuration is required; follow the custom-runner documentation. | + +### CI Infrastructure recommendations + +- Avoid full clones, especially in large repositories. +- Keep the number of jobs within reason because concurrent capacity is shared across repositories. +- Public repositories must not depend on private repositories. +- Use the job-sizing dashboard to select appropriate CPU and memory for Kubernetes jobs. +- On Windows, pre-pull frequently used images into the runner AMI. +- Structure Dockerfiles so frequently changing layers appear late and do not invalidate stable, expensive layers. + +## Migration principles + +- Keep Azure DevOps running during the migration. +- Add GitLab jobs without changing existing Azure behavior. +- Reuse NUKE targets as the unit of work. +- Do not rewrite build logic in YAML. +- Keep platform-specific setup in shared templates. +- Use shallow fetches where possible. +- Build and test platforms in parallel. +- Publish artifacts from each platform build and consume them through explicit `needs`. +- Gate PoC jobs with a reversible variable. +- Measure correctness, duration, queueing, artifact size, and runner utilization before expanding the matrix. + +## Completed foundation work + +### Windows compiler parallelism and memory + +[PR #8925](https://github.com/DataDog/dd-trace-dotnet/pull/8925) is merged. + +It: + +- Enables MSVC multiprocessor compilation. +- Caps `CL_MPCount` at the runner's 16 logical processors. +- Raises the Windows build-container limit from 10 GB to 20 GB. +- Logs runner RAM and logical processors. +- Measures peak container memory during the build. + +The runner reported 31.2 GB total RAM and 26.6 GB free before the build. Peak usage with `/MP16` was approximately 9.1 GiB, leaving reasonable headroom within the 20 GB limit. + +Measured results: + +| Configuration | NUKE | Native compilation | End-to-end | Peak container memory | +| --- | ---: | ---: | ---: | ---: | +| Baseline, MP disabled / 10 GB | 31:45 | 24:52 | ~40 min | Not measured | +| MP2 / 10 GB | 22:27 | 14:40 | ~29–31 min | Not measured | +| MP3 / 10 GB | Not measured | Not measured | ~24–28 min | Not measured | +| MP4 / 16 GB | 15:00–15:54 | Not measured | ~17–23 min | Not measured | +| MP8 / 16 GB | 13:50 | 6:01 | 23:27 | Not measured | +| MP8 / 20 GB | 13:11 | 5:44 | 21:31 | 8.686 GiB | +| MP16 / 20 GB | 11:37 | 4:37 | 19:05–19:54 | 9.105 GiB | + +Native compilation detail: + +| Configuration | Tracer | Loader | Profiler | +| --- | ---: | ---: | ---: | +| Baseline | 8:38 | 2:46 | 13:28 | +| MP2 / 10 GB | 5:15 | 1:40 | 7:45 | +| MP8 / 16 GB | 2:21 | 0:42 | 2:58 | +| MP8 / 20 GB | 2:13 | 0:41 | 2:50 | +| MP16 / 20 GB | 1:50 | 0:33 | 2:14 | + +The optimized job is approximately 50% faster end to end than the original baseline. Run-to-run variance remains significant and was dominated by Docker image pulling. + +### CI log cleanup + +[PR #8948](https://github.com/DataDog/dd-trace-dotnet/pull/8948) is merged. + +It: + +- Treats vendored spdlog/fmt headers as MSVC external headers. +- Uses `ExternalWarningLevel=TurnOffAllWarnings`, which generates `/external:W0`. +- Preserves warnings from Datadog and PPDB sources. +- Keeps external-header settings in the relevant `.vcxproj` files. +- Filters routine NuGet restore progress in CI while preserving warnings and errors. +- Keeps verbose NuGet output locally. +- Enables process output before attaching the NUKE custom logger. +- Applies the filter for `IsServerBuild || IsGitlab`, because the Windows Docker invocation does not expose all variables NUKE normally uses to detect GitLab. + +Measured log reduction: + +| Metric | Original | Intermediate | Latest | +| --- | ---: | ---: | ---: | +| Lines | 17,699 | 7,894 | 3,585 | +| Size | 2.55 MiB | 1.21 MiB | 0.46 MiB | +| Targeted NuGet messages | 5,475 | 4,308 | 0 | +| Compiler warnings | 1,036 | 116 | 116 | +| Errors | 0 | 0 | 0 | + +The final log is approximately 80% shorter and is easier to inspect when later migration work fails. + +### Content-addressed Windows build image + +The Windows image is stored at: + +```text +registry.ddbuild.io/ci/dd-trace-dotnet/dd-trace-dotnet-docker-build +``` + +`tracer/build/_build/docker/gitlab/compute-image-hash.ps1`: + +1. Lists each file directly under `tracer/build/_build/docker/gitlab`, sorted by name. +2. Computes the SHA-256 of each file. +3. Concatenates those hashes. +4. Hashes the concatenated value. +5. Uses the first 12 lowercase characters as the image tag. + +The current `master` checkout produces `814a0509e85a`. + +The normal GitLab build verifies that the exact hash-tagged image exists. If it does not, the job fails with instructions to run the manual `build-windows-ci-image` job. That job builds and pushes both the content-addressed tag and `:latest`. Consumers always use the content-addressed tag. + +This deliberately prevents a Dockerfile change from silently running against an old image. + +### Windows native unit-test migration + +The Windows native unit tests have now been added to GitLab and have completed successfully. They run in the existing `build` job immediately after the production build, reusing the same checkout and native compilation state. This matches Azure's build-then-test pattern without transferring intermediate objects between runners. + +The build job: + +- Uses `windows-v2:2022` and the content-addressed Windows build image. +- Passes `--NugetPackageDirectory c:\mnt\packages`. Without this argument, restoring an individual profiler test `.vcxproj` fails because NuGet cannot infer a packages or solution directory. +- Passes the same repository-mounted package directory to the build, test, and packaging invocations. Each invocation uses a short-lived Docker container, so packages restored to a container's default user profile would otherwise disappear before the next phase. +- Retains the 20 GB container limit and `/MP16` settings. +- Compiles and runs the tracer, native loader, and profiler native test suites. +- Runs packaging and signing targets only after the native tests pass. +- Publishes tracer/loader results from `artifacts/build_data/tests` and profiler results from `profiler/build_data/tests` through GitLab's JUnit report support, including when the job fails. +- Gates the later `publish` stage as part of the build job itself. + +The first separated run took approximately 12 minutes. Its original packaging-artifact download took only about 3.5 seconds and did not contain reusable native compilation state. NUKE spent 1:10 restoring and 9:31 compiling/running the native suites; 8:38 of that was compilation. The log confirmed that the profiler, loader, and tracer production projects were rebuilt through project references. This motivated transferring the selective native build-state artifact described above. + +A first attempt used a commit-scoped GitLab cache. Creating it added approximately 2:03 to the build, and the runner reported that no shared-cache URL was configured, so the cache remained local to one EC2 runner. It was removed because a downstream test job is not guaranteed to use the same runner. Regular GitLab artifacts provide the required cross-runner guarantee. + +The first run with the selective native build-state artifact transferred 631 files with an uncompressed size of 4.68 GB. Downloading and extracting it took approximately 65 seconds. Reusing the production outputs reduced the native-test NUKE invocation from 9:31 to 7:53 and the complete job from approximately 11:59 to 11:13. The largest improvement was in the profiler test compilation, which fell from 5:29 to 3:59. The net improvement for this single consumer was therefore only about 46 seconds, but the same build state can later be reused by Windows unit, integration, and packaging jobs. + +Azure provides the closest equivalent through `build-windows-working-directory`. The Windows tracer build publishes the entire working directory, and numerous downstream jobs restore it into `$(System.DefaultWorkingDirectory)`. A measured Azure artifact contained 27,222 files and was shown as 4,483 MB in the artifact UI; a downstream download reported 4,724.1 MB of total content. Azure transferred 2,548.5 MB physically, saved 2,119.1 MB through compression, and reused 56.5 MB from the runner's local cache. + +Azure's upload uses chunk-level deduplication. For the measured run, it processed 3,893,529,926 source bytes and reported 7,781.7 MB of deduplicated content, but uploaded only 0.5 MB physically because nearly all chunks already existed in Azure's artifact store. This makes repeated Azure uploads exceptionally cheap, but does not eliminate the multi-gigabyte download on a downstream runner. GitLab's regular job artifact provides reliable cross-runner handoff but does not provide equivalent deduplicated-upload behavior. + +The large artifact is CI build state, not a release deliverable. Azure separately publishes the curated `windows-tracer-home` (98 MB), `windows-profiler-home` (15 MB), profiler symbols, and later ZIP, MSI, and NuGet artifacts. GitLab should preserve the same distinction: retain the native bin/object state only as short-lived input for downstream CI jobs, while keeping final monitoring-home, symbols, installer, and package outputs as the artifacts used by release workflows. + +Based on these measurements, the separate `test-native-windows` job and its selective 4.68 GB build-state artifact were removed. A single native-test consumer gained only about 46 seconds after paying the transfer cost, while Azure already avoids this handoff by running native tests in its producing build jobs. GitLab now does the same. The measurements remain useful when deciding what build state future managed, integration, or packaging jobs actually require, but native bin/object trees are no longer published solely for native tests. + +The first complete run of the final build → test → package topology succeeded. Directing all three short-lived containers to `c:\mnt\packages` fixed the prior `NETSDK1064` failure: the test compilation resolved `System.Collections` 4.3.0 and the other managed dependencies from the shared workspace. The timing breakdown was: + +| Phase | NUKE duration | +| --- | ---: | +| Production build | 10:34 | +| Native tests | 6:26 | +| Packaging and signing | 2:15 | +| Total NUKE work | 19:15 | + +The complete GitLab job took approximately 35:32. Pulling the hash-tagged Windows image took about 13:49, from the start of the automatic pull until Docker reported the newer image downloaded. This confirms that image availability, rather than build/test/package execution, remains the dominant avoidable cost. The build container peaked at 8.628 GiB, 43.1% of its 20 GB limit. Artifact publication remained small and fast: GitLab found 135 release artifact entries, four tracer/loader XML reports, and two profiler XML reports; both archive and JUnit uploads succeeded. + +The next Windows PoC adds managed unit tests for `net8.0` and `net48` to the same build job, after native tests and before packaging. These two frameworks cover a modern .NET runtime and the distinct .NET Framework execution path without immediately multiplying the Windows image-pull cost across the full Azure matrix. Each framework runs `BuildManagedUnitTests` followed by `RunManagedUnitTests` against the existing workspace and shared `c:\mnt\packages` directory. + +Managed test results use TRX rather than JUnit XML. Because each NUKE invocation cleans the shared results and logs directories, the job copies them into framework-specific directories under `artifacts/build_data/managed-unit-tests/` before starting the next framework. They are retained as ordinary job artifacts for the PoC. Converting TRX to JUnit, or otherwise surfacing the managed results directly in GitLab's test-report UI, remains pending before expanding the complete matrix. + +The first managed-test attempts exposed a broader restore boundary. The solution-level Windows `NuGet.exe restore` did not populate every SDK-style `PackageReference` dependency needed by the managed test graph in the shared directory. This first appeared as a missing `StyleCop.Analyzers.Unstable` package in `BuildRunnerTool`; after restoring that project directly, compilation reached another missing package, `Microsoft.NET.ILLink.Analyzers`, through `Datadog.Trace.Tools.dd_dotnet`. The project-specific workaround was replaced with one `RestoreManagedUnitTestPackages` prerequisite. When an explicit `NugetPackageDirectory` is provided, it uses `dotnet restore` on the solution before `BuildRunnerTool` and `CompileManagedUnitTests`. GitLab supplies `c:\mnt\packages`, so the complete SDK-style package graph survives the short-lived container boundary. Without that parameter the prerequisite is skipped, preserving Azure and local behavior. + +The first run also revealed that the configured JUnit paths pointed at `build-out`, while NUKE writes results to `artifacts/build_data/tests` and `profiler/build_data/tests`. The paths are corrected. Native-loader x64 and x86 previously wrote the same filename; the filename now uses the loop architecture so both reports are retained. + +The Windows commands are: + +```text +CompileTracerNativeTests +RunTracerNativeTests +CompileNativeLoaderNativeTests +RunNativeLoaderNativeTests +CompileProfilerNativeTests +RunProfilerNativeTests +``` + +Windows coverage compared with Azure: + +| Coverage | GitLab | Azure DevOps | +| --- | --- | --- | +| Runner definition | `windows-v2:2022` | `azure-managed-windows-x64-2` | +| Tracer native tests | x64 and x86 | x64 and x86 | +| Native loader tests | x64 and x86 | x64 and x86 | +| Profiler native tests | x64 and x86 | x64 and x86 | +| Windows ARM64/ARM64EC | Not tested | Not tested | +| Multiple Windows versions | No | No | + +NUKE defaults to an x64 target on these runners, but the Windows native-test targets explicitly compile and execute both x64 and x86. GitLab therefore has parity with Azure for these Windows native suites. + +Structural differences remain: + +- Azure runs tracer/loader tests in the Windows tracer build job and profiler tests in a separate Windows profiler build job. GitLab runs all three suites after its unified Windows build in the same job. +- Azure retries each native-test command once. GitLab now forwards `DD_LOGGER_DD_API_KEY` to the test container but does not yet provide Azure's task-level retry. +- Azure additionally runs native tests on Linux x64 for glibc/CentOS 7 and musl/Alpine. Its Linux profiler jobs also run the native wrapper tests. This Linux coverage has not yet moved to GitLab. +- Azure's Linux ARM64 and macOS build stages do not currently invoke these native unit-test targets. + +## Active Windows image work and findings + +### Pre-pull the image into the Windows runner AMI + +Repeated logs show that pulling the dd-trace Windows image takes approximately 7–9 minutes. The four base Windows layers are already present, but the remaining dd-trace layers are downloaded for each fresh runner. + +The CI Infrastructure team confirmed that the intended optimization is: + +```text +ci-platform-machine-images/packer/windows/scripts/base/pre-pull-docker-images.ps1 +``` + +Findings: + +- The script runs while the Windows AMI is baked. +- The Windows root volume defaults to 300 GB, so the additional cached image should not create a material storage problem. +- Pre-pull the exact content tag consumed by CI, not only `:latest`. +- The exact tag must be refreshed whenever a file under `tracer/build/_build/docker/gitlab` changes. +- A modified copy currently exists at `citemp/packer/windows/scripts/base/pre-pull-docker-images.ps1`. +- That copy currently references `814a0509e85a`. +- Do not submit that tag unchanged after PR #8962 or another image change merges. Recompute the hash, build and publish the new image, and then update the AMI PR. + +Expected benefit: remove approximately 7–9 minutes from a fresh Windows job. Based on measured runs, that could reduce a roughly 19–24 minute job to approximately 12–17 minutes, subject to runner and build variance. + +### Pre-install vcpkg helper tools + +[PR #8962](https://github.com/DataDog/dd-trace-dotnet/pull/8962) is open. + +The PR: + +- Installs and bootstraps vcpkg in the GitLab Windows image. +- Pre-fetches helper tools such as Git, CMake, 7-Zip, PowerShell Core, and Ninja. +- Uses vcpkg's default downloads root so the pre-fetched helper tools are reused. +- Avoids downloading the helper toolchain on every GitLab build. + +The latest GitLab build, Azure Windows tracer build, and Azure Windows profiler build are green. + +Review findings: + +- The vcpkg version remains duplicated between `gitlab.windows.dockerfile` and `Build.Steps.cs`. +- `GetVcpkg()` prefers any `vcpkg.exe` on `PATH` and does not validate its version. A future one-sided version bump could therefore make GitLab and fallback builds use different versions. +- Prefer a single checked-in version source, for example `vcpkg-version.txt`, read by both NUKE and the Docker installation. +- `install_vcpkg.ps1` downloads a GitHub tag archive without verifying a checksum. Pinning and verifying SHA-256 would align it with the other image installers. +- A checksum calculated from the same first download pins the observed bytes but is not independent publisher verification. +- GitHub-generated source archives may eventually change their compressed byte layout. For stronger reproducibility, pin the tag's commit SHA as well; a release asset with a publisher-provided digest would be preferable if one exists. +- Removing the explicit downloads root changes all Windows builds, not only GitLab. The successful Azure Windows jobs reduce the immediate compatibility risk, but the behavior should remain documented. +- Pre-downloading libdatadog itself remains a possible later optimization and is intentionally out of scope for PR #8962. + +## Proposed GitLab file layout + +Use one file per pipeline phase, with all platforms represented as separate jobs or matrices inside that file: + +```text +.gitlab-ci.yml +.gitlab/ + ci/ + templates.yml + images.yml + build.yml + test-unit.yml + test-integration.yml + package.yml + test-smoke.yml + benchmarks/ +``` + +Responsibilities: + +- `templates.yml`: common defaults, rules, runner tags, and per-platform NUKE skeletons. +- `images.yml`: image hash calculation and image-build jobs. +- `build.yml`: platform builds and their artifacts. +- `test-unit.yml`: native and managed unit-test jobs that consume platform build artifacts. +- `test-integration.yml`: integration and profiler-integration tests. +- `package.yml`: packaging and signing. +- `test-smoke.yml`: artifact smoke tests. + +Naming convention: + +```text +:[:][:][:] +``` + +Examples: + +```text +gl-build:windows:x64 +gl-build:linux:arm64:musl +gl-test-unit:linux:x64:glibc:net8.0 +``` + +During parallel validation, use `gl-` stage and job prefixes so the new jobs cannot collide with existing jobs. Gate the PoC with: + +```yaml +GITLAB_POC: "true" +``` + +The variable should default to enabled during validation but allow the PoC jobs to be disabled without removing YAML. + +## Phase 1: builds and unit tests + +### Scope + +- Build Windows x64. +- Build Linux x64 on glibc and musl. +- Build Linux ARM64 on glibc and musl. +- Build macOS ARM64. +- Produce platform monitoring-home artifacts without signing or packaging. +- Run native unit tests explicitly in downstream jobs when build artifacts can be reused efficiently. +- Run managed unit tests in downstream jobs by target framework. + +Out of scope: + +- Signing and MSI/deb/rpm packaging. +- S3 and OCI publishing. +- R2R variants. +- Sample and debug builds. +- IIS and Azure Functions tests. +- Integration and smoke tests. +- macOS AMD64. + +### Top-level changes + +Modify `.gitlab-ci.yml` additively: + +- Include `.gitlab/ci/templates.yml`. +- Include `.gitlab/ci/images.yml`. +- Include `.gitlab/ci/build.yml`. +- Include `.gitlab/ci/test-unit.yml`. +- Add `gl-images`, `gl-build`, and `gl-test-unit` stages. +- Add the `GITLAB_POC` variable. +- Preserve all existing jobs and stages. + +### Shared templates + +Proposed defaults: + +```yaml +.dd-default: + interruptible: true + variables: + GIT_DEPTH: 20 + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: recursive + +.dd-rules-poc: + rules: + - if: '$GITLAB_POC == "true"' + when: on_success + - when: never +``` + +Before adopting recursive submodules, verify that the repository requires them; avoid unnecessary clone work. + +Add `.dd-nuke-windows`, `.dd-nuke-linux`, and `.dd-nuke-macos` templates with their images, runner tags, setup, and common artifact handling. + +### Image jobs + +Windows: + +- Reuse `tracer/build/_build/docker/gitlab/compute-image-hash.ps1`. +- Keep or refactor the existing manual `build-windows-ci-image` job. +- Continue failing consumers when their expected content-addressed image is absent. + +Linux: + +- Add `tracer/build/_build/docker/compute-linux-image-hash.sh`. +- Build Debian, Alpine, CentOS 7, universal, and Alpine ARM64 variants as required. +- Use `arch:amd64` or `arch:arm64` runners with remote BuildKit. +- Publish content-addressed images under a repository such as: + + ```text + registry.ddbuild.io/ci/dd-trace-dotnet/dd-trace-dotnet-linux-build-: + ``` + +The exact Linux image set should be derived from Phase 1 jobs first; avoid building unused variants. + +### Build jobs + +| Job | Runner | Image | Principal NUKE targets | +| --- | --- | --- | --- | +| `gl-build:windows:x64` | `windows-v2:2022` | Windows content-hash image | `BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet` | +| `gl-build:linux:x64:glibc` | `arch:amd64` | Debian build image | `Clean CompileManagedLoader BuildNativeTracerHome BuildManagedTracerHome BuildNativeLoader BuildNativeWrapper BuildDdDotnet ExtractDebugInfoLinux BuildProfilerHome` | +| `gl-build:linux:x64:musl` | `arch:amd64` | Alpine build image | Same platform-appropriate build targets | +| `gl-build:linux:arm64:glibc` | `arch:arm64` | ARM64 Debian build image | Same platform-appropriate build targets | +| `gl-build:linux:arm64:musl` | `arch:arm64` | ARM64 Alpine build image | Same platform-appropriate build targets | +| `gl-build:macos:arm64` | `macos:sonoma-arm64` | Native runner | `CreateRequiredDirectories CompileManagedLoader BuildNativeTracerHome BuildManagedTracerHome BuildNativeLoader` | + +Linux jobs may use a matrix over architecture and libc when the image, runner tag, artifact name, and `needs` mapping remain understandable. + +#### Native-test requirement + +Do not assume that `BuildTracerHome` or `BuildProfilerHome` automatically runs the native unit tests. Azure invokes them explicitly. + +Windows parity requires at least: + +```text +CompileTracerNativeTests +RunTracerNativeTests +CompileNativeLoaderNativeTests +RunNativeLoaderNativeTests +CompileProfilerNativeTests +RunProfilerNativeTests +``` + +The existing `test-native-windows` PoC now provides this Windows parity in a downstream job. It should be moved into the proposed Phase 1 file layout rather than reimplemented. + +Additional native-test jobs required for current Azure parity: + +| Job | Coverage | Principal NUKE targets | +| --- | --- | --- | +| `gl-build:windows:x64` | Windows x64 and x86, inline after the build | The six tracer, loader, and profiler targets listed above | +| `gl-test-native:linux:x64:glibc` | Linux x64 on glibc | Tracer, profiler, native loader, and native wrapper compile/run targets | +| `gl-test-native:linux:x64:musl` | Linux x64 on musl | The same Linux native targets in Alpine | + +Azure does not currently invoke these native unit-test targets in its Linux ARM64 or macOS build stages, so those platforms are not required for native-test parity in Phase 1. + +### Build artifacts + +Publish the minimum artifacts required by downstream tests: + +- `monitoringHome/` +- Required managed build outputs under `tracer/bin/` +- Required profiler outputs +- Required shared/native outputs +- Native test reports from the producing Windows build job and downstream Linux native-test jobs + +Do not publish broad working directories by default. Measure artifact size and transfer time, then add only missing paths required by downstream jobs. Native tests currently run in the build job and therefore do not require native bin/object artifacts. + +Current Windows measurements: + +| Measurement | GitLab selective build state | Azure `build-windows-working-directory` | +| --- | ---: | ---: | +| Files | 631 | 27,222 | +| Reported/uncompressed size | 4.68 GB | 4,483 MB in the artifact UI | +| Downstream total content | 4.68 GB uncompressed | 4,724.1 MB | +| Physical downstream download | Not reported separately | 2,548.5 MB | +| Download and extraction | Approximately 65 seconds | Duration not available in the captured excerpt | +| Current native-test benefit | Approximately 46 seconds end-to-end | Tests run in the producing build jobs | + +Azure's artifact is broader, while the tested GitLab artifact selected native tracer, loader, and profiler state and excluded vcpkg. It was removed from the current pipeline because the transfer had limited value for one native-test consumer. Reconsider a minimal build-state artifact only when additional downstream Windows jobs demonstrate that its shared benefit exceeds its upload, storage, and download cost. + +### Managed unit-test jobs + +| Job pattern | Matrix | Runner | Dependency | NUKE target | +| --- | --- | --- | --- | --- | +| `gl-test-unit:windows:x64` | `net48`, `net8.0`, `net9.0`, `net10.0` | `windows-v2:2022` | `gl-build:windows:x64` | `RunManagedUnitTests --framework $FRAMEWORK` | +| `gl-test-unit:linux` | arch × libc × `net8.0`, `net9.0`, `net10.0` | matching `arch:*` | matching Linux build | `RunManagedUnitTests --framework $FRAMEWORK` | +| `gl-test-unit:macos:arm64` | `net8.0`, `net9.0`, `net10.0` | `macos:sonoma-arm64` | `gl-build:macos:arm64` | `RunManagedUnitTests --framework $FRAMEWORK` | + +Publish JUnit-compatible XML with `artifacts:reports:junit` so failures appear directly on the merge request. + +Initial Phase 1 budget: + +- 6 build jobs. +- 2 Linux native-unit-test jobs; Windows native tests run inline in the Windows build job. +- 4 Windows managed-unit-test jobs. +- 12 Linux managed-unit-test jobs. +- 3 macOS managed-unit-test jobs. +- 28 total jobs. + +Validate the budget with the CI Infrastructure team because concurrency is shared and job count alone does not capture queueing or CPU usage. + +### Phase 1 success criteria + +- All 28 jobs pass on representative merge requests. +- Existing Azure and GitLab jobs remain unchanged and green. +- Each build publishes the expected platform artifacts. +- Each managed test job publishes a visible JUnit report. +- GitLab and Azure monitoring-home outputs are equivalent for the same commit, allowing timestamps and explicitly documented signing differences. +- Compare artifacts manually for approximately the first ten successful PoC runs. +- No unexplained binary or file-set differences. +- Windows build remains below 25 minutes before AMI pre-pull and targets approximately 12–17 minutes after pre-pull. +- Each Linux build cell targets 25 minutes or less. +- macOS build targets 30 minutes or less. +- Total build and unit-test wall clock targets 75 minutes or less, including queueing. +- The PoC can be disabled through `GITLAB_POC` without reverting code. + +## Phase 2 + +No separate Phase 2 is currently required. Managed and native unit tests are part of Phase 1. + +## Phase 3: integration tests + +Add `.gitlab/ci/test-integration.yml`. + +Initial direction: + +- Linux integration jobs use `docker-in-docker:amd64` and `docker-in-docker:arm64`. +- Reuse `docker-compose.yml`, Testcontainers, and existing NUKE targets. +- Start with matrices over architecture, libc, and framework. +- Keep Windows IIS tests separate because they mutate machine state. +- Keep Windows Azure Functions tests separate. +- Add macOS integration tests per supported framework. +- Mirror profiler integration tests on Windows and Linux. + +The proposed coarse matrix favors fewer, longer jobs to reduce shared-runner contention. Measure real durations before committing to it. Split by area only where a job approaches its timeout or becomes too costly to retry. + +## Phase 3.5: packaging + +Add `.gitlab/ci/package.yml`. + +Windows: + +- Consume `gl-build:windows:x64`. +- Run packaging and signing targets such as `PackageTracerHome`, `PublishFleetInstaller`, `SignDlls`, and `SignMsi`. +- Produce MSI, symbols, tracer home, Fleet Installer, and related artifacts with Azure-equivalent naming and retention. + +Linux: + +- Consume the matching architecture/libc build. +- Produce tar.gz, deb, and rpm artifacts as applicable. +- Verify whether current Azure packages are GPG-signed and reproduce that behavior. + +Packaging must be complete before smoke tests that consume installed artifacts. + +## Phase 4: smoke tests + +Add `.gitlab/ci/test-smoke.yml`. + +Candidate categories: + +- Installer +- NuGet +- Tool +- Trimming +- Self-instrumentation +- Fleet Installer +- `dd-dotnet` + +Use matrices over architecture, libc, and .NET version where appropriate. + +- Linux smoke tests use Docker-in-Docker microVMs. +- Windows smoke tests use `windows-v2:2022`. +- macOS initially covers applicable tool tests on `macos:sonoma-arm64`. + +Start with approximately 10–12 job definitions and measure the expanded matrix before enabling all cells by default. + +## Phase 5: validation and switchover + +Run Azure and GitLab in parallel for at least four weeks. + +Proposed confidence requirement: + +- At least 95% of merge requests in the window have Azure and GitLab in agreement: both green or both red for equivalent work. +- Classify disagreements as infrastructure, flaky test, configuration difference, artifact difference, or genuine product regression. +- A comparison job or scheduled process should publish agreement and duration metrics to Datadog. + +Suggested removal order: + +1. Stop Azure packaging stages after artifact equivalence is established. +2. Stop Azure build stages after GitLab artifacts are proven consumable. +3. Stop Azure test stages incrementally: unit, integration, then smoke. +4. Re-point OCI, serverless, and publishing jobs to GitLab artifacts. +5. Remove `.gitlab/download-single-step-artifacts.sh` and `.gitlab/download-serverless-artifacts.sh`. +6. Remove Azure status-reporting hooks. +7. Delete `.azure-pipelines/`. +8. Drop the temporary `gl-` prefixes after the new pipeline becomes authoritative. + +Do not create a test-stage outage between stopping Azure builds and migrating tests. Either keep Azure builds until their dependent tests move, or explicitly validate cross-CI artifact consumption first. + +## Verification procedure + +### Local command sanity + +Windows: + +```powershell +docker run --rm ` + -v "${PWD}:C:\src" ` + $WINDOWS_BUILD_IMAGE ` + BuildTracerHome BuildProfilerHome BuildNativeLoader BuildDdDotnet +``` + +Run the six explicit Windows native-test targets immediately after the production build in the same job when validating Phase 1 parity. + +Linux: + +```bash +docker run --rm -v "$PWD:/src" \ + ./tracer/build.sh +``` + +Run the x64 glibc and musl native-test targets separately to match Azure. ARM64 native unit tests are not part of current Azure parity. + +macOS: + +```bash +./tracer/build.sh \ + CreateRequiredDirectories \ + CompileManagedLoader \ + BuildNativeTracerHome \ + BuildManagedTracerHome \ + BuildNativeLoader +``` + +No macOS native unit-test targets are required for current Azure parity. + +### Pipeline dry run + +- Push a branch with `GITLAB_POC=true`. +- Confirm all 28 Phase 1 jobs are created. +- Confirm each job selects the intended runner and image. +- Confirm build artifacts are available through `needs`. +- Confirm test reports appear on the merge request. +- Record execution and queue duration separately. + +### Artifact comparison + +For the same commit: + +1. Download GitLab and Azure artifacts. +2. Normalize timestamps and known signing differences. +3. Compare file lists, binary contents, symbols, permissions, and expected executable formats. +4. Treat every other difference as a migration defect until explained. + +### AMI pre-pull validation + +Before updating the runner AMI: + +- Record the exact requested image tag. +- Record the `Unable to find image ... locally` timestamp. +- Record the final `Downloaded newer image` timestamp. + +After deployment: + +- Confirm the exact hash-tagged image exists locally. +- Confirm no layers are downloaded before the build. +- Compare fresh-runner duration with the prior 7–9 minute pull. + +### Azure unchanged + +- Confirm `.azure-pipelines/ultimate-pipeline.yml` remains functionally unchanged during Phase 1. +- Confirm existing GitLab build, publish, benchmark, OCI, and serverless jobs remain functional. +- Run a clean `master` build after foundational image changes. + +## Open decisions and risks + +### Immediate + +- Merge or otherwise resolve PR #8962 before finalizing the AMI pre-pull tag. +- Decide whether to centralize the vcpkg version before merging PR #8962. +- Decide whether to pin and verify the vcpkg archive checksum. +- Recompute and publish the final Windows image hash. +- Update the `ci-platform-machine-images` PR to pre-pull that exact tag. +- Make `docker pull` failures explicit in the pre-pull script: + + ```powershell + docker pull $image + if ($LASTEXITCODE -ne 0) { + throw "Failed to pull Docker image '$image' (exit code $LASTEXITCODE)." + } + ``` + +- Run the Windows 2022 Packer build manually on the AMI PR. +- Deploy the AMI and validate the measured pull-time reduction. +- Decide whether the Windows Packer job must become a required PR check. It is manual on PR branches and automatic on `main`. + +### Phase 1 design + +- Confirm the concurrent-job and runner-capacity budget with CI Infrastructure. +- Confirm whether recursive submodules are required. +- Map the remaining Azure Linux x64 native-test and native-wrapper commands to explicit GitLab jobs; the Windows mapping is complete and green. +- Define the minimum artifact set for each downstream test job. +- Confirm GitLab `needs` behavior and artifact naming for parallel matrices. +- Decide whether Linux needs every proposed image variant in Phase 1. +- Define Linux image ownership, rebuild procedure, hash inputs, and publication permissions. +- Decide how image-building jobs authenticate without creating private-repository dependencies that violate public-repository guidance. + +### Later phases + +- Decide when to add the Linux x64 R2R variant. +- Decide whether samples are built once or inside integration jobs. +- Replace Azure's generated-variable matrices with static YAML or a small generated child pipeline. +- Confirm Linux package-signing parity. +- Re-evaluate Dockerfile layer splitting if AMI pre-pull and vcpkg initialization do not meet the Windows target. +- Consider pre-caching libdatadog after measuring its remaining contribution. +- Define status-reporting and Datadog metrics for Azure/GitLab agreement. + +## Critical files + +### Proposed new files + +- `.gitlab/ci/templates.yml` +- `.gitlab/ci/images.yml` +- `.gitlab/ci/build.yml` +- `.gitlab/ci/test-unit.yml` +- `.gitlab/ci/test-integration.yml` +- `.gitlab/ci/package.yml` +- `.gitlab/ci/test-smoke.yml` +- `tracer/build/_build/docker/compute-linux-image-hash.sh` + +### Files modified during the migration + +- `.gitlab-ci.yml` +- `tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md` +- `tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile` +- `tracer/build/_build/docker/gitlab/compute-image-hash.ps1` +- `tracer/build/_build/docker/gitlab/entrypoint.bat` +- `ci-platform-machine-images/packer/windows/scripts/base/pre-pull-docker-images.ps1` in the separate runner-image repository + +### Existing sources of truth + +- `.azure-pipelines/ultimate-pipeline.yml`: current parity reference until Phase 5. +- `tracer/build/_build/Build.cs` and related partial files: NUKE orchestration and target source of truth. +- `tracer/build.sh` and `tracer/build.cmd`: platform launchers. +- Existing Dockerfiles and Docker Compose files: test and packaging environment definitions. diff --git a/tracer/build/_build/Build.Shared.Steps.cs b/tracer/build/_build/Build.Shared.Steps.cs index 3824655ac070..582b78bb3177 100644 --- a/tracer/build/_build/Build.Shared.Steps.cs +++ b/tracer/build/_build/Build.Shared.Steps.cs @@ -81,7 +81,7 @@ partial class Build foreach (var architecture in ArchitecturesForPlatformForTracer) { var workingDirectory = GetNativeOutputDirectory(NativeLoaderTestsProject.Name) / BuildConfiguration / architecture.ToString(); - var testsResultFile = BuildDataDirectory / "tests" / $"{FileNames.NativeLoaderTests}.Results.{BuildConfiguration}.{TargetPlatform}.xml"; + var testsResultFile = BuildDataDirectory / "tests" / $"{FileNames.NativeLoaderTests}.Results.{BuildConfiguration}.{architecture}.xml"; var exePath = workingDirectory / $"{FileNames.NativeLoaderTests}.exe"; var testExe = ToolResolver.GetLocalTool(exePath); testExe($"--gtest_output=xml:{testsResultFile}", workingDirectory: workingDirectory); diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 62a6094f8b7c..2f0747ebddc8 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -324,6 +324,22 @@ bool RequiresThoroughTesting() } }); + Target RestoreManagedUnitTestPackages => _ => _ + .Unlisted() + .Before(BuildRunnerTool, CompileManagedUnitTests) + .OnlyWhenDynamic(() => !string.IsNullOrEmpty(NugetPackageDirectory)) + .Executes(() => + { + // NuGet.exe restore does not fully populate all SDK-style PackageReference + // dependencies required by the managed test graph. GitLab runs the build and + // tests in separate containers, so restore them into the mounted package directory. + DotNetRestore(s => s + .SetProjectFile(Solution) + .SetVerbosity(DotNetVerbosity.Minimal) + .SetProperty("configuration", BuildConfiguration.ToString()) + .SetPackageDirectory(NugetPackageDirectory)); + }); + Target CompileTracerNativeSrcWindows => _ => _ .Unlisted() .After(CompileManagedLoader) diff --git a/tracer/build/_build/Build.cs b/tracer/build/_build/Build.cs index 6b496feeaf4d..bb2e7e3c2517 100644 --- a/tracer/build/_build/Build.cs +++ b/tracer/build/_build/Build.cs @@ -276,6 +276,7 @@ void DeleteReparsePoints(string path) .Description("Builds the managed unit tests") .After(Clean, BuildTracerHome, BuildProfilerHome) .DependsOn(CreateRequiredDirectories) + .DependsOn(RestoreManagedUnitTestPackages) .DependsOn(BuildRunnerTool) .DependsOn(CompileManagedUnitTests);