Pre-install and initialize vcpkg in Windows GitLab (#8962)

## Summary of changes

Initialize vcpkg and pre-install its dependencies in Windows GitLab
image

## Reason for change

vcpkg is used to download libdatadog, but the first time it runs, it
downloads and installs all its own dependencies (git, 7zip, powershell
etc). That can cause flake, and is unecessary work to do on every build

## Implementation details

- Preinstall the vcpkg depencencies in the GitLab windows image
- Stop specifying a custom download location for these deps, and instead
use the default (I don't see a good reason _not_ to do this). Note that
this does not change where _packages_ are downloaded to, only vcpkg
dependencies.
- We can re-use this inside the Azure VMs, but for now, those are
unchanged (requires updating the VM scripts and rebuilding them)

## Test coverage

The PR is the test. Rebuilt the image, and built in Gitlab and verified
we're no longer downloading vcpkg. Azure Devops is unchanged.

## Other details

We _could_ (arguably _should_) pre-download the libdatadog version too,
and cache that in the image, and then rebuild the image when we need to
bump the version. However, that made the solution more complicated,
needing to copy files around, include it in the build context etc, so
kept the scope limited for now. We can always extend it to include
libdatadog later.

#incident-57303

Author: andrewlock

profiler/Directory.Build.targets

@@ -5,7 +5,8 @@
     <VcpkgDepsDir>$(VcpkgManifestRoot)\artifacts\deps\vcpkg\</VcpkgDepsDir>
     <!-- trying mimicking what's vcpkg does by default but redirecting where we want the final files -->
     <VcpkgInstalledDir>$(VcpkgDepsDir)$(VcpkgTriplet)</VcpkgInstalledDir>
-    <VcpkgAdditionalInstallOptions>--downloads-root $(VcpkgIntermediateFolder)/downloads --x-packages-root $(VcpkgIntermediateFolder)/packages --x-buildtrees-root $(VcpkgIntermediateFolder)/buildtrees  --clean-after-build</VcpkgAdditionalInstallOptions>
+    <!-- We leave downloads-root (for vcpkg deps) as the default, so that we can pre-populate it once (e.g. in CI)-->
+    <VcpkgAdditionalInstallOptions>--x-packages-root $(VcpkgIntermediateFolder)/packages --x-buildtrees-root $(VcpkgIntermediateFolder)/buildtrees  --clean-after-build</VcpkgAdditionalInstallOptions>
   </PropertyGroup>
   <ItemDefinitionGroup Condition="'$(VcpkgUseStatic)' == 'true' and '$(VcpkgEnabled)' == 'true' and '$(MSBuildProjectExtension)' == '.vcxproj'">
     <Link>

tracer/build/_build/Build.Steps.cs

@@ -661,7 +661,6 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
                             // note that the following are all quoted when entered into the vcpkg call itself
                             // this is necessary to support cases where we have a space in the folder name
                             var installRoot = $@"{BuildArtifactsDirectory}\deps\vcpkg\{triplet}";
-                            var downloads = $@"{BuildArtifactsDirectory}\obj\vcpkg\downloads";
                             var packages = $@"{BuildArtifactsDirectory}\obj\vcpkg\packages";
                             var buildTrees = $@"{BuildArtifactsDirectory}\obj\vcpkg\buildtrees";
 
@@ -672,8 +671,9 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo
                                 try
                                 {
                                     Logger.Information($"Attempt {attempt}: Running vcpkg install for {triplet}...");
-                                    // This big line is the same generated by VS when installing libdatadog while building the profiler
-                                    vcpkg($@"install --x-wait-for-lock --triplet ""{triplet}"" --vcpkg-root ""{vcpkgRoot}"" ""--x-manifest-root={RootDirectory}"" ""--x-install-root={installRoot}"" --downloads-root ""{downloads}"" --x-packages-root ""{packages}"" --x-buildtrees-root ""{buildTrees}"" --clean-after-build");
+                                    // This big line is the same generated by VS when installing libdatadog while building the profiler.
+                                    // We leave downloads-root (for vcpkg deps) as the default, so that we can pre-populate it once (e.g. in CI)
+                                    vcpkg($@"install --x-wait-for-lock --triplet ""{triplet}"" --vcpkg-root ""{vcpkgRoot}"" ""--x-manifest-root={RootDirectory}"" ""--x-install-root={installRoot}"" --x-packages-root ""{packages}"" --x-buildtrees-root ""{buildTrees}"" --clean-after-build");
                                     Logger.Information($"vcpkg install succeeded on attempt {attempt}.");
                                     break; // Exit loop on success
                                 }
@@ -2883,6 +2883,7 @@ private async Task DownloadAndExtractVcpkg(AbsolutePath destinationFolder)
         var keepTrying = true;
         var vcpkgZip = TempDirectory / "vcpkg.zip";
         using var client = new HttpClient();
+        // Keep this version in sync with VCPKG_VERSION in gitlab.windows.dockerfile
         const string vcpkgVersion = "2024.11.16";
         while (keepTrying)
         {

tracer/build/_build/docker/gitlab/UPDATING_IMAGE.md

@@ -41,6 +41,12 @@ Once `build-windows-ci-image` is green, re-run the `build:` job (or any other Wi
 - The `:latest` tag is **only** used to seed the Docker build cache on the next rebuild. Nothing in the pipeline consumes `:latest` as an input at runtime; the consumer always pins to the content hash.
 - Changes to `compute-image-hash.ps1` itself also invalidate the hash, because the script hashes its own directory.
 
+### vcpkg toolchain
+
+`install_vcpkg.ps1` installs vcpkg into `C:\vcpkg` and pre-fetches the helper tools vcpkg would otherwise download on every build (e.g, `cmake`, `7zip` etc). 
+
+- Keep `VCPKG_VERSION` in `gitlab.windows.dockerfile` in sync with the `vcpkgVersion` constant in `tracer/build/_build/Build.Steps.cs`, so the pre-fetched tool versions match what the build's vcpkg expects.
+
 ### Example PR
 
 - [dd-trace-dotnet#7492](https://github.com/DataDog/dd-trace-dotnet/pull/7492) (earlier, local-build flow — for structural reference)

tracer/build/_build/docker/gitlab/gitlab.windows.dockerfile

@@ -54,6 +54,15 @@ RUN powershell -Command .\install_java.ps1
 # Install Windows Code Signer
 COPY --from=registry.ddbuild.io/windows-code-signer/go:v0.7.0 c:/windows-code-signer/windows-code-signer.exe c:/devtools/windows-code-signer.exe
 
+# Install vcpkg and pre-fetch its helper toolchain. 
+# Keep VCPKG_VERSION in sync with the vcpkgVersion constant in
+# Build.Steps.cs. See UPDATING_IMAGE.md.
+ENV VCPKG_VERSION="2024.11.16" \
+    VCPKG_ROOT="C:\vcpkg"
+
+COPY install_vcpkg.ps1 .
+RUN powershell -Command .\install_vcpkg.ps1 -Version $ENV:VCPKG_VERSION -InstallRoot $ENV:VCPKG_ROOT
+
 # Copy everything else
 COPY . .
 ENTRYPOINT ["/entrypoint.bat"]

tracer/build/_build/docker/gitlab/install_vcpkg.ps1

@@ -0,0 +1,50 @@
+param (
+    [Parameter(Mandatory=$true)][string]$Version,
+    [Parameter(Mandatory=$false)][string]$InstallRoot = "C:\vcpkg"
+)
+
+# Installs and bootstraps vcpkg into a fixed location, then pre-fetches the helper tools it downloads
+# on first use (cmake, 7zip, powershell-core, ninja). GetVcpkg() in tracer/build/_build/Build.Steps.cs
+# finds vcpkg.exe on PATH, and the build no longer relocates vcpkg's downloads root, so these
+# pre-fetched tools (under $InstallRoot\downloads\tools) are reused instead of downloaded on every
+# build. Keep $Version in sync with the vcpkgVersion constant in Build.Steps.cs so the pre-fetched
+# tool versions match the ones the build's vcpkg expects.
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+$out = "$($PSScriptRoot)\vcpkg.zip"
+$url = "https://github.com/microsoft/vcpkg/archive/refs/tags/$Version.zip"
+
+Write-Host -ForegroundColor Green "Downloading vcpkg $Version from $url to $out"
+(New-Object System.Net.WebClient).DownloadFile($url, $out)
+
+Write-Host -ForegroundColor Green "Extracting $out"
+$parent = Split-Path -Parent $InstallRoot
+Expand-Archive -Path $out -DestinationPath $parent -Force
+Remove-Item $out
+
+# The archive expands to a "vcpkg-<version>" folder; rename it to the fixed install root.
+if (Test-Path $InstallRoot) { Remove-Item -Recurse -Force $InstallRoot }
+Rename-Item -Path (Join-Path $parent "vcpkg-$Version") -NewName (Split-Path -Leaf $InstallRoot)
+
+Write-Host -ForegroundColor Green "Bootstrapping vcpkg"
+& "$InstallRoot\bootstrap-vcpkg.bat" -disableMetrics
+if ($LASTEXITCODE -ne 0) { throw "bootstrap-vcpkg.bat failed with exit code $LASTEXITCODE" }
+
+# Add vcpkg to the machine PATH so it is resolved by ToolPathResolver.GetPathExecutable at build time.
+[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";$InstallRoot", [System.EnvironmentVariableTarget]::Machine)
+
+# Pre-fetch the helper tools vcpkg would otherwise download on first use. These land under
+# $InstallRoot\downloads\tools, which is the default downloads root the build uses now that it no
+# longer overrides --downloads-root.
+foreach ($tool in @('git', 'cmake', '7zip', 'powershell-core', 'ninja')) {
+    Write-Host -ForegroundColor Green "Pre-fetching vcpkg tool: $tool"
+    & "$InstallRoot\vcpkg.exe" fetch $tool
+    if ($LASTEXITCODE -ne 0) { throw "vcpkg fetch $tool failed with exit code $LASTEXITCODE" }
+}
+
+Write-Host -ForegroundColor Green "Installed vcpkg $Version to $InstallRoot"
+& "$InstallRoot\vcpkg.exe" version