Skip to content

Commit 5f3c2a8

Browse files
igoragoliigoragoli
authored
Move microbench AMI provision file into this repo, stop cloning benchmarking-platform (#8458)
## Summary of changes - Move `ephemeral-infra/ami.yaml` from the `dd-trace-dotnet/micro` branch of the `benchmarking-platform` repo into `.gitlab/benchmarks/microbenchmarks/infrastructure/ami.yml`, alongside the existing `instance.yml` - Update the `build-dd-trace-dotnet-microbenchmarks-ami` job to reference the local file directly via `PROVISION_FILE`, removing the need to clone the external branch ## Reason for change The `dd-trace-dotnet/micro` branch in `benchmarking-platform` is being removed. The AMI provision file is now co-located with the rest of the microbenchmark infrastructure config in this repo. ## Implementation details The `run-benchmarks` job already references `instance.yml` via a local path. The AMI build job now follows the same pattern for `ami.yml`. ## Test coverage AMI build with the current setup: https://gitlab.ddbuild.io/DataDog/apm-reliability/dd-trace-dotnet/-/jobs/1600227613 ## Other details None.
1 parent 9863b8b commit 5f3c2a8

2 files changed

Lines changed: 302 additions & 17 deletions

File tree

‎.gitlab/benchmarks/microbenchmarks.yml‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -28,43 +28,27 @@ build-dd-trace-dotnet-microbenchmarks-ami:
2828
allow_failure: true
2929
when: manual
3030
image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:dd-trace-dotnet-micro
31-
id_tokens:
32-
DDOCTOSTS_ID_TOKEN:
33-
aud: dd-octo-sts
3431
variables:
3532
# Adding managed-by:bp-infra lets us filter CI runner metrics to monitor bp-infra-managed jobs.
3633
# See https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/2889355451/Gitlab+Runner+fork#Metrics
3734
DD_METRICS_TAGS: "managed-by:bp-infra"
3835

39-
# Allows ephemeral instances to read content from dd-trace-dotnet
40-
# This is not strictly necessary in the current AMI build
41-
DDOCTOSTS_SCOPE: "DataDog/dd-trace-dotnet"
42-
DDOCTOSTS_POLICY: "gitlab.github-access.read-contents"
43-
4436
AWS_REGION: "us-east-1"
4537

46-
# Branch containing a provision for building the AMI
47-
BP_INFRA_BENCHMARKING_PLATFORM_BRANCH: "dd-trace-dotnet/micro"
48-
49-
PROVISION_FILE: "platform/ephemeral-infra/ami.yaml"
38+
PROVISION_FILE: ".gitlab/benchmarks/microbenchmarks/infrastructure/ami.yml"
5039

5140
# Where AMI creation artifacts will be stored
5241
BP_INFRA_ARTIFACTS_BUCKET_NAME: "windows-benchmarking-results-us-east-1"
5342

5443
# Whether to cleanup instances after building the AMI, since the AMI is
5544
# based on an instance that is created in this job
5645
CLEANUP: "true"
57-
before_script:
58-
- !reference [.dd-octo-sts-setup, before_script]
5946
script:
60-
- git clone --branch $BP_INFRA_BENCHMARKING_PLATFORM_BRANCH https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/${GITLAB_GROUP}/benchmarking-platform platform
61-
- echo "GITHUB_TOKEN=$(cat /tmp/github-token)" > .env
6247
- CLEANUP_ARG=$([[ "$CLEANUP" == "false" ]] && echo "--no-cleanup" || echo "")
6348
- |
6449
bp-infra launch --region "${AWS_REGION}" --os "windows" \
6550
--provision "${PROVISION_FILE}" \
6651
--bypass-stack-destroy \
67-
--env .env \
6852
$CLEANUP_ARG
6953
after_script:
7054
# Makes sure the instance is cleaned up.

‎.gitlab/benchmarks/microbenchmarks/infrastructure/ami.yml‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
name: &provision_name "dd-trace-dotnet-microbenchmarks-ami"
2+
3+
init_environment:
4+
# Defines the .NET SDK versions to install on the AMI
5+
6+
# These versions are chosen to align with the dd-trace-dotnet repository's supported SDK versions: https://github.com/DataDog/dd-trace-dotnet/blob/a4a4ce0822735b3c8cf6862bcabce81fc0959753/tracer/build/_build/Build.cs#L598-L602
7+
8+
# It includes the final End-of-Life (EOL) versions for .NET Core 3.1 and .NET 6.0 along with the latest .NET version.
9+
DOTNET_SDK_VERSIONS: "6.0.428,3.1.426,10.0.100"
10+
11+
source_ami:
12+
# From https://github.com/DataDog/benchmarking-platform-tools/tree/main/bp-infra/src/provisions/windows/apm-windows-tweaked-metal.yaml
13+
ami_id_ssm_parameter: "/windows-benchmarking/apm-windows-tweaked-metal-ami-id"
14+
15+
create_ami:
16+
ami_id_ssm_parameter: "/windows-benchmarking/dd-trace-dotnet-microbenchmarks-ami-id"
17+
18+
tags:
19+
Name: *provision_name
20+
21+
provision_steps:
22+
- install_dotnet
23+
- install_vs
24+
- test_setup
25+
26+
install_dotnet:
27+
populate_env: true
28+
remote_command: |
29+
if (-not $env:DOTNET_SDK_VERSIONS) {
30+
Write-Error "DOTNET_SDK_VERSIONS is not set."
31+
exit 1
32+
}
33+
34+
$installDir = "C:\dotnet"
35+
$scriptUrl = "https://dot.net/v1/dotnet-install.ps1"
36+
37+
$versions = $env:DOTNET_SDK_VERSIONS.Split(',')
38+
foreach ($version in $versions) {
39+
try {
40+
Write-Output "Installing .NET SDK $version..."
41+
&([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version $version -InstallDir $installDir;
42+
} catch {
43+
Write-Output "Failed to install .NET SDK $version. Continuing with other installations..."
44+
}
45+
}
46+
47+
[Environment]::SetEnvironmentVariable("PATH", "$installDir;" + [Environment]::GetEnvironmentVariable("PATH", [EnvironmentVariableTarget]::Machine), [EnvironmentVariableTarget]::Machine)
48+
49+
Write-Output "Installation completed for $env:DOTNET_SDK_VERSIONS."
50+
51+
install_vs:
52+
populate_env: true
53+
remote_command: |
54+
$version="17.14.36310.24"
55+
$exeUrl="https://download.visualstudio.microsoft.com/download/pr/ae7ac791-9759-4076-bba7-47ff510c57af/a783199025439d65f310bff041e278b966a6dbed8dbcd7fc96b55389f574ef41/vs_BuildTools.exe"
56+
$installDir="c:\devtools\vstudio"
57+
58+
# Enabled TLS12
59+
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
60+
61+
Write-Output -ForegroundColor Green "Installing Visual Studio $($version) from $($exeUrl)"
62+
63+
# Script directory is $PSScriptRoot
64+
$exeOut = "$($PSScriptRoot)\vs_buildtools.exe"
65+
66+
Write-Output -ForegroundColor Green Downloading $exeUrl to $exeOut
67+
(New-Object System.Net.WebClient).DownloadFile($exeUrl, $exeOut)
68+
69+
# Write file size to make sure it worked
70+
Write-Output -ForegroundColor Green "File size is $((get-item $exeOut).length)"
71+
72+
$VSPackages = @(
73+
"Microsoft.VisualStudio.Component.Roslyn.Compiler",
74+
"Microsoft.Component.MSBuild",
75+
"Microsoft.VisualStudio.Component.CoreBuildTools",
76+
"Microsoft.VisualStudio.Workload.MSBuildTools",
77+
"Microsoft.VisualStudio.Component.Windows10SDK",
78+
"Microsoft.VisualStudio.Component.VC.CoreBuildTools",
79+
"Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
80+
"Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
81+
"Microsoft.VisualStudio.Component.VC.CMake.Project",
82+
"Microsoft.VisualStudio.Component.TestTools.BuildTools",
83+
"Microsoft.VisualStudio.Component.VC.ATL",
84+
"Microsoft.Net.Component.4.8.SDK",
85+
"Microsoft.Net.Component.4.7.2.TargetingPack",
86+
"Microsoft.VisualStudio.Component.VC.ASAN",
87+
"Microsoft.VisualStudio.Component.TextTemplating",
88+
"Microsoft.VisualStudio.Component.VC.CoreIde",
89+
"Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
90+
"Microsoft.VisualStudio.Component.Windows10SDK.20348",
91+
"Microsoft.VisualStudio.Component.Windows10SDK.19041",
92+
"Microsoft.VisualStudio.Component.Windows10SDK.18362",
93+
"Microsoft.VisualStudio.Workload.VCTools",
94+
"Microsoft.VisualStudio.Component.NuGet.BuildTools",
95+
"Microsoft.VisualStudio.Web.BuildTools.ComponentGroup",
96+
"Microsoft.Net.ComponentGroup.DevelopmentPrerequisites",
97+
"Microsoft.VisualStudio.Component.TypeScript.TSServer",
98+
"Microsoft.Net.Component.4.8.TargetingPack",
99+
"Microsoft.Net.ComponentGroup.4.8.DeveloperTools",
100+
"Microsoft.NetCore.Component.Runtime.9.0",
101+
"Microsoft.NetCore.Component.Runtime.8.0",
102+
"Microsoft.NetCore.Component.SDK",
103+
"Microsoft.Component.ClickOnce.MSBuild",
104+
"Microsoft.VisualStudio.Wcf.BuildTools.ComponentGroup",
105+
"Microsoft.VisualStudio.Component.WebDeploy",
106+
"Microsoft.Net.Component.4.6.2.TargetingPack",
107+
"Microsoft.Net.Component.4.7.TargetingPack",
108+
"Microsoft.VisualStudio.Workload.WebBuildTools",
109+
"Microsoft.Net.Component.4.6.1.TargetingPack",
110+
"Microsoft.VisualStudio.Workload.ManagedDesktopBuildTools",
111+
"Microsoft.Net.Component.4.6.TargetingPack",
112+
"Microsoft.VisualStudio.Component.NuGet",
113+
"Microsoft.VisualStudio.Component.VC.Tools.ARM64EC",
114+
"Microsoft.VisualStudio.Component.VC.Tools.ARM64",
115+
"Microsoft.VisualStudio.Component.Roslyn.LanguageServices",
116+
"Component.LinuxBuildTools",
117+
"Microsoft.VisualStudio.Workload.LinuxBuildTools",
118+
"Microsoft.VisualStudio.Component.VC.ATL.ARM64",
119+
"Microsoft.VisualStudio.Component.VC.Runtimes.x86.x64.Spectre",
120+
"Microsoft.NetCore.Component.Runtime.6.0",
121+
"Microsoft.NetCore.Component.Runtime.7.0",
122+
"Microsoft.VisualStudio.Component.VC.v141.ATL",
123+
"Microsoft.VisualStudio.Component.VC.14.29.16.11.x86.x64",
124+
"Microsoft.VisualStudio.Component.VC.14.29.16.11.ATL",
125+
"Microsoft.VisualStudio.Component.VC.14.35.17.5.ARM64",
126+
"Microsoft.VisualStudio.Component.VC.14.35.17.5.x86.x64",
127+
"Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL.ARM64",
128+
"Microsoft.VisualStudio.Component.VC.14.35.17.5.ATL",
129+
"Microsoft.VisualStudio.Component.VC.14.35.17.5.CLI.Support"
130+
)
131+
132+
$VSPackageListParam = $VSPackages -join " --add "
133+
$ArgList = "--wait --norestart --nocache --installPath `"$($installDir)`" --add $VSPackageListParam"
134+
if(-not $NoQuiet){
135+
$ArgList = "--quiet $ArgList"
136+
}
137+
$processparams = @{
138+
FilePath = $exeOut
139+
NoNewWindow = $true
140+
Wait = $true
141+
ArgumentList = $ArgList
142+
}
143+
Start-Process @processparams
144+
145+
setx VSTUDIO_ROOT "$installDir"
146+
[Environment]::SetEnvironmentVariable("VSTUDIO_ROOT", "$installDir", [System.EnvironmentVariableTarget]::Machine)
147+
148+
# Add SDK added above to path for signtool
149+
[Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) + ";${env:ProgramFiles(x86)}\Windows Kits\10\bin\10.0.18362.0\x64", [System.EnvironmentVariableTarget]::Machine)
150+
151+
Remove-Item $exeOut
152+
Write-Output -ForegroundColor Green "Visual Studio $($version) installed successfully."
153+
154+
test_setup:
155+
remote_command: |
156+
# Future improvements: Write this inside a bp-pwsh-utils module, and import it.
157+
function Test-Setup {
158+
$checks = @(
159+
{
160+
$dotnetPath = "C:\dotnet\dotnet.exe"
161+
if (Test-Path $dotnetPath) {
162+
$version = (& $dotnetPath --version)
163+
"[PASS] .NET is installed (Version at $dotnetPath is $version. Multiple versions might be installed)"
164+
}
165+
else {
166+
"[FAIL] .NET is not installed"
167+
}
168+
},
169+
{
170+
$vsPath = "C:\devtools\vstudio\Common7\Tools\VsDevCmd.bat"
171+
if (Test-Path $vsPath) {
172+
$output = (& $vsPath)
173+
$version = $output | Select-String -Pattern "Visual Studio (.*)" | Select-Object -First 1 | ForEach-Object { $_.Matches.Groups[1].Value }
174+
"[PASS] Visual Studio is installed (Version: $version)"
175+
}
176+
else {
177+
"[FAIL] Visual Studio is not installed"
178+
}
179+
},
180+
{
181+
$currentSize = (Get-Partition -DriveLetter C).Size
182+
$maxSize = (Get-PartitionSupportedSize -DriveLetter C).SizeMax
183+
if (($maxSize - $currentSize) -lt 1MB) {
184+
"[PASS] C: drive is at maximum size ($([math]::Round($currentSize/1GB, 2)) GB)"
185+
}
186+
else {
187+
"[FAIL] C: drive is not at maximum size (Current: $([math]::Round($currentSize/1GB, 2)) GB, Max: $([math]::Round($maxSize/1GB, 2)) GB)"
188+
}
189+
},
190+
{
191+
$sshService = Get-Service -Name sshd -ErrorAction SilentlyContinue
192+
if ($null -ne $sshService -and $sshService.Status -eq 'Running') {
193+
"[PASS] SSH Server is installed and running"
194+
}
195+
else {
196+
"[FAIL] SSH Server is not installed or not running"
197+
}
198+
},
199+
{
200+
$gitPath = "C:\Program Files\Git\cmd\git.exe"
201+
if (Test-Path $gitPath) {
202+
try {
203+
$version = (& $gitPath --version).Replace('git version ', '')
204+
"[PASS] Git is installed (Version: $version)"
205+
}
206+
catch {
207+
"[FAIL] Git exists but version check failed"
208+
}
209+
}
210+
else {
211+
"[FAIL] Git is not installed"
212+
}
213+
},
214+
{
215+
$bpRunner = Get-Command bp-runner -ErrorAction SilentlyContinue
216+
if ($bpRunner) {
217+
try {
218+
$version = (bp-runner --version 2>&1)
219+
"[PASS] bp-runner is installed (Version: $version)"
220+
}
221+
catch {
222+
"[FAIL] bp-runner exists but version check failed"
223+
}
224+
}
225+
else {
226+
"[FAIL] bp-runner is not installed"
227+
}
228+
}
229+
)
230+
$checks | ForEach-Object { & $_ }
231+
}
232+
233+
function Test-PowercfgSetting {
234+
param(
235+
[string]$subGroup,
236+
[string]$setting,
237+
[int]$expectedAC,
238+
[int]$expectedDC
239+
)
240+
$expectedACHex = "0x{0:X8}" -f $expectedAC
241+
$expectedDCHex = "0x{0:X8}" -f $expectedDC
242+
$currentAC = powercfg /query SCHEME_CURRENT $subGroup $setting | Select-String -Pattern "Power Setting Index" | Select-Object -First 1
243+
$currentDC = powercfg /query SCHEME_CURRENT $subGroup $setting | Select-String -Pattern "Power Setting Index" | Select-Object -Last 1
244+
$currentACValue = $currentAC -replace '.*: ', ''
245+
$currentDCValue = $currentDC -replace '.*: ', ''
246+
247+
if ($currentACValue -eq $expectedACHex -and $currentDCValue -eq $expectedDCHex) {
248+
Write-Output "[PASS] $setting is correctly set to $expectedACHex (AC) and $expectedDCHex (DC)"
249+
}
250+
else {
251+
Write-Output "[FAIL] $setting is not set correctly! Expected $expectedACHex (AC), $expectedDCHex (DC), but got $currentACValue (AC), $currentDCValue (DC)"
252+
}
253+
}
254+
255+
function Test-HyperThreading {
256+
$physicalCores = (Get-CimInstance -Class Win32_Processor | Measure-Object -Property NumberOfCores -Sum).Sum
257+
$logicalCores = (Get-CimInstance -Class Win32_Processor | Measure-Object -Property NumberOfLogicalProcessors -Sum).Sum
258+
259+
$htEnabled = $logicalCores -gt $physicalCores
260+
if ($htEnabled -eq $false) {
261+
Write-Output "[PASS] Hyper-threading is correctly disabled. Physical Cores: $physicalCores, Logical Cores: $logicalCores"
262+
}
263+
else {
264+
Write-Output "[FAIL] Hyper-threading is enabled but should be disabled! Physical Cores: $physicalCores, Logical Cores: $logicalCores"
265+
}
266+
}
267+
268+
function Test-Tweaks {
269+
$checks = @(
270+
{ Test-PowercfgSetting SUB_PROCESSOR PERFBOOSTMODE 0 0 },
271+
{ powercfg /getactivescheme },
272+
{ Test-PowercfgSetting SUB_PROCESSOR PROCFREQMAX 2500 2500 },
273+
{ Test-PowercfgSetting SUB_PROCESSOR IDLESTATEMAX 1 1 },
274+
{ Test-PowercfgSetting SUB_PROCESSOR IDLEDISABLE 1 1 },
275+
{ Test-PowercfgSetting SUB_PROCESSOR PROCTHROTTLEMIN 100 100 },
276+
{ Test-PowercfgSetting SUB_PROCESSOR PROCTHROTTLEMAX 100 100 },
277+
{ Test-PowercfgSetting SUB_PROCESSOR PROCTHROTTLEMIN1 100 100 },
278+
{ Test-PowercfgSetting SUB_PROCESSOR PROCTHROTTLEMAX1 100 100 },
279+
{ Test-PowercfgSetting SUB_PROCESSOR PERFBOOSTMODE 0 0 },
280+
{ Test-HyperThreading }
281+
)
282+
$checks | ForEach-Object { & $_ }
283+
}
284+
285+
$artifactsDir = "C:\ProvisionStepsArtifacts"
286+
New-Item -Path $artifactsDir -ItemType Directory -Force
287+
288+
Write-Output "Verifying tweaks for benchmark repeatability..."
289+
$tweaksOutput = Test-Tweaks
290+
$tweaksOutput | Out-File "$artifactsDir\TestTweaksResults.log"
291+
$tweaksOutput
292+
293+
Write-Output "Verifying setup..."
294+
$setupOutput = Test-Setup
295+
$setupOutput | Out-File "$artifactsDir\TestSetupResults.log"
296+
$setupOutput
297+
298+
if (($tweaksOutput -like "*FAIL*") -or ($setupOutput -like "*FAIL*")) {
299+
Write-Output "Setup test failed"
300+
exit 1
301+
}

0 commit comments

Comments
 (0)