Skip to content

Commit f4ebe34

Browse files
surajssdsurajssd
committed
Add NVIDIA DCGM packages and repository support
- Add 5 new NVIDIA DCGM packages to components.json: - datacenter-gpu-manager-4-core - datacenter-gpu-manager-4-cuda12 - datacenter-gpu-manager-4-proprietary - datacenter-gpu-manager-4-proprietary-cuda12 - datacenter-gpu-manager-exporter - Configure Renovate to auto-update NVIDIA packages: - Add custom datasources for nvidia-deb2204 and nvidia-deb2404 - Add regex matchers for NVIDIA packages in components.json - Point to NVIDIA CUDA repository package indexes - Add NVIDIA repository setup functions: - updateAptWithNvidiaPkg() for Ubuntu to add NVIDIA CUDA repos - updateDnfWithNvidiaPkg() for AzureLinux to add NVIDIA repos - Support for both x86_64 and arm64/sbsa architectures - Add error codes for NVIDIA GPG key and repo file download timeouts - Update VHD build process to download NVIDIA DCGM packages during image creation Signed-off-by: Suraj Deshmukh <suraj.deshmukh@microsoft.com>
1 parent 4c23611 commit f4ebe34

7 files changed

Lines changed: 358 additions & 41 deletions

File tree

‎.github/renovate.json‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@
7373
"versioning": "regex:^v(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-(?<prerelease>\\d+)$",
7474
"ignoreUnstable": false
7575
},
76-
7776
{
7877
"matchDatasources": [
7978
"docker"
@@ -570,6 +569,20 @@
570569
"versioningTemplate": "deb",
571570
"autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=production, os=ubuntu, release=22.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
572571
},
572+
{
573+
"customType": "regex",
574+
"description": "auto update Nvidia packages for OS ubuntu 22.04 in components.json",
575+
"managerFilePatterns": [
576+
"/parts/common/components.json/"
577+
],
578+
"matchStringsStrategy": "any",
579+
"matchStrings": [
580+
"\"renovateTag\":\\s*\"name=(?<packageName>[^\"]+), repository=production, os=ubuntu, release=22\\.04\",\\s*\"latestVersion\":\\s*\"(?<currentValue>[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?<depType>[^\"]+)\")?"
581+
],
582+
"datasourceTemplate": "custom.nvidia-deb2204",
583+
"versioningTemplate": "deb",
584+
"autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=production, os=ubuntu, release=22.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
585+
},
573586
{
574587
"customType": "regex",
575588
"description": "auto update packages for OS ubuntu 24.04 in components.json",
@@ -584,6 +597,20 @@
584597
"versioningTemplate": "deb",
585598
"autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=production, os=ubuntu, release=24.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
586599
},
600+
{
601+
"customType": "regex",
602+
"description": "auto update Nvidia packages for OS ubuntu 24.04 in components.json",
603+
"managerFilePatterns": [
604+
"/parts/common/components.json/"
605+
],
606+
"matchStringsStrategy": "any",
607+
"matchStrings": [
608+
"\"renovateTag\":\\s*\"name=(?<packageName>[^\"]+), repository=production, os=ubuntu, release=24\\.04\",\\s*\"latestVersion\":\\s*\"(?<currentValue>[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?<depType>[^\"]+)\")?"
609+
],
610+
"datasourceTemplate": "custom.nvidia-deb2404",
611+
"versioningTemplate": "deb",
612+
"autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, repository=production, os=ubuntu, release=24.04\",\n \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
613+
},
587614
{
588615
"customType": "regex",
589616
"description": "auto update packages for OS ubuntu 24.04 (test registry) in components.json",
@@ -651,13 +678,27 @@
651678
"{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}"
652679
]
653680
},
681+
"nvidia-deb2204": {
682+
"defaultRegistryUrlTemplate": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/Packages",
683+
"format": "plain",
684+
"transformTemplates": [
685+
"{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}"
686+
]
687+
},
654688
"deb2404": {
655689
"defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/24.04/prod/dists/noble/main/binary-amd64/Packages",
656690
"format": "plain",
657691
"transformTemplates": [
658692
"{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}"
659693
]
660694
},
695+
"nvidia-deb2404": {
696+
"defaultRegistryUrlTemplate": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/Packages",
697+
"format": "plain",
698+
"transformTemplates": [
699+
"{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}"
700+
]
701+
},
661702
"deb2404-test": {
662703
"defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/24.04/prod/dists/testing/main/binary-amd64/Packages",
663704
"format": "plain",

‎parts/common/components.json‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1475,6 +1475,176 @@
14751475
}
14761476
}
14771477
}
1478+
},
1479+
{
1480+
"name": "datacenter-gpu-manager-4-core",
1481+
"downloadLocation": "/opt/dcgm/downloads",
1482+
"downloadURIs": {
1483+
"ubuntu": {
1484+
"r2404": {
1485+
"versionsV2": [
1486+
{
1487+
"renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=24.04",
1488+
"latestVersion": "1:4.4.1-1"
1489+
}
1490+
]
1491+
},
1492+
"r2204": {
1493+
"versionsV2": [
1494+
{
1495+
"renovateTag": "name=datacenter-gpu-manager-4-core, repository=nvidia, os=ubuntu, release=22.04",
1496+
"latestVersion": "1:4.4.1-1"
1497+
}
1498+
]
1499+
}
1500+
},
1501+
"azurelinux": {
1502+
"v3.0": {
1503+
"versionsV2": [
1504+
{
1505+
"renovateTag": "RPM_registry=https://developer.download.nvidia.com/compute/cuda/repos/azl3/x86_64/repodata, name=datacenter-gpu-manager-4-core, repository=nvidia, os=azurelinux, release=3.0",
1506+
"latestVersion": "1:4.4.1-1"
1507+
}
1508+
]
1509+
}
1510+
}
1511+
}
1512+
},
1513+
{
1514+
"name": "datacenter-gpu-manager-4-cuda12",
1515+
"downloadLocation": "/opt/dcgm/downloads",
1516+
"downloadURIs": {
1517+
"ubuntu": {
1518+
"r2404": {
1519+
"versionsV2": [
1520+
{
1521+
"renovateTag": "name=datacenter-gpu-manager-4-cuda12, repository=nvidia, os=ubuntu, release=24.04",
1522+
"latestVersion": "1:4.4.1-1"
1523+
}
1524+
]
1525+
},
1526+
"r2204": {
1527+
"versionsV2": [
1528+
{
1529+
"renovateTag": "name=datacenter-gpu-manager-4-cuda12, repository=nvidia, os=ubuntu, release=22.04",
1530+
"latestVersion": "1:4.4.1-1"
1531+
}
1532+
]
1533+
}
1534+
},
1535+
"azurelinux": {
1536+
"v3.0": {
1537+
"versionsV2": [
1538+
{
1539+
"renovateTag": "RPM_registry=https://developer.download.nvidia.com/compute/cuda/repos/azl3/x86_64/repodata, name=datacenter-gpu-manager-4-cuda12, repository=nvidia, os=azurelinux, release=3.0",
1540+
"latestVersion": "1:4.4.1-1"
1541+
}
1542+
]
1543+
}
1544+
}
1545+
}
1546+
},
1547+
{
1548+
"name": "datacenter-gpu-manager-4-proprietary",
1549+
"downloadLocation": "/opt/dcgm/downloads",
1550+
"downloadURIs": {
1551+
"ubuntu": {
1552+
"r2404": {
1553+
"versionsV2": [
1554+
{
1555+
"renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=24.04",
1556+
"latestVersion": "1:4.4.1-1"
1557+
}
1558+
]
1559+
},
1560+
"r2204": {
1561+
"versionsV2": [
1562+
{
1563+
"renovateTag": "name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=ubuntu, release=22.04",
1564+
"latestVersion": "1:4.4.1-1"
1565+
}
1566+
]
1567+
}
1568+
},
1569+
"azurelinux": {
1570+
"v3.0": {
1571+
"versionsV2": [
1572+
{
1573+
"renovateTag": "RPM_registry=https://developer.download.nvidia.com/compute/cuda/repos/azl3/x86_64/repodata, name=datacenter-gpu-manager-4-proprietary, repository=nvidia, os=azurelinux, release=3.0",
1574+
"latestVersion": "1:4.4.1-1"
1575+
}
1576+
]
1577+
}
1578+
}
1579+
}
1580+
},
1581+
{
1582+
"name": "datacenter-gpu-manager-4-proprietary-cuda12",
1583+
"downloadLocation": "/opt/dcgm/downloads",
1584+
"downloadURIs": {
1585+
"ubuntu": {
1586+
"r2404": {
1587+
"versionsV2": [
1588+
{
1589+
"renovateTag": "name=datacenter-gpu-manager-4-proprietary-cuda12, repository=nvidia, os=ubuntu, release=24.04",
1590+
"latestVersion": "1:4.4.1-1"
1591+
}
1592+
]
1593+
},
1594+
"r2204": {
1595+
"versionsV2": [
1596+
{
1597+
"renovateTag": "name=datacenter-gpu-manager-4-proprietary-cuda12, repository=nvidia, os=ubuntu, release=22.04",
1598+
"latestVersion": "1:4.4.1-1"
1599+
}
1600+
]
1601+
}
1602+
},
1603+
"azurelinux": {
1604+
"v3.0": {
1605+
"versionsV2": [
1606+
{
1607+
"renovateTag": "RPM_registry=https://developer.download.nvidia.com/compute/cuda/repos/azl3/x86_64/repodata, name=datacenter-gpu-manager-4-proprietary-cuda12, repository=nvidia, os=azurelinux, release=3.0",
1608+
"latestVersion": "1:4.4.1-1"
1609+
}
1610+
]
1611+
}
1612+
}
1613+
}
1614+
},
1615+
{
1616+
"name": "datacenter-gpu-manager-exporter",
1617+
"downloadLocation": "/opt/dcgm/downloads",
1618+
"downloadURIs": {
1619+
"ubuntu": {
1620+
"r2404": {
1621+
"versionsV2": [
1622+
{
1623+
"renovateTag": "name=datacenter-gpu-manager-exporter, repository=nvidia, os=ubuntu, release=24.04",
1624+
"latestVersion": "4.5.2-1"
1625+
}
1626+
]
1627+
},
1628+
"r2204": {
1629+
"versionsV2": [
1630+
{
1631+
"renovateTag": "name=datacenter-gpu-manager-exporter, repository=nvidia, os=ubuntu, release=22.04",
1632+
"latestVersion": "4.5.2-1"
1633+
}
1634+
]
1635+
}
1636+
},
1637+
"azurelinux": {
1638+
"v3.0": {
1639+
"versionsV2": [
1640+
{
1641+
"renovateTag": "RPM_registry=https://developer.download.nvidia.com/compute/cuda/repos/azl3/x86_64/repodata, name=datacenter-gpu-manager-exporter, repository=nvidia, os=azurelinux, release=3.0",
1642+
"latestVersion": "4.5.2-1"
1643+
}
1644+
]
1645+
}
1646+
}
1647+
}
14781648
}
14791649
],
14801650
"OCIArtifacts": [

‎parts/linux/cloud-init/artifacts/cse_helpers.sh‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,8 @@ ERR_SECURE_TLS_BOOTSTRAP_START_FAILURE=220 # Error starting the secure TLS boots
137137

138138
ERR_CLOUD_INIT_FAILED=223 # Error indicating that cloud-init returned exit code 1 in cse_cmd.sh
139139
ERR_NVIDIA_DRIVER_INSTALL=224 # Error determining if nvidia driver install should be skipped
140+
ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT=225 # Timeout waiting for NVIDIA GPG key download
141+
ERR_NVIDIA_AZURELINUX_REPO_FILE_DOWNLOAD_TIMEOUT=226 # Timeout waiting for NVIDIA AzureLinux repo file download
140142

141143
# For both Ubuntu and Mariner, /etc/*-release should exist.
142144
# For unit tests, the OS and OS_VERSION will be set in the unit test script.
@@ -185,7 +187,7 @@ AKS_AAD_SERVER_APP_ID="6dae42f8-4368-4678-94ff-3960e28e3630"
185187
# Checks if the elapsed time since CSEStartTime exceeds 13 minutes.
186188
# That value is based on the global CSE timeout which is set to 15 minutes - majority of CSE executions succeed or fail very fast, meaning we can exit slightly before the global timeout without affecting the overall CSE execution.
187189
# Global cse timeout is set in cse_start.sh: `timeout -k5s 15m /bin/bash /opt/azure/containers/provision.sh`
188-
# Long running functions can use this helper to gracefully handle global CSE timeout, avoiding exiting with 124 error code without extra context.
190+
# Long running functions can use this helper to gracefully handle global CSE timeout, avoiding exiting with 124 error code without extra context.
189191
check_cse_timeout() {
190192
shouldLog="${1:-true}"
191193
maxDurationSeconds=780 # 780 seconds = 13 minutes
@@ -224,11 +226,11 @@ _retrycmd_internal() {
224226
exitStatus=$?
225227

226228
if [ "$exitStatus" -eq 0 ]; then
227-
break
229+
break
228230
fi
229231

230232
# Check if CSE timeout is approaching - exit early to avoid 124 exit code from the global timeout
231-
if ! check_cse_timeout "$shouldLog"; then
233+
if ! check_cse_timeout "$shouldLog"; then
232234
echo "CSE timeout approaching, exiting early." >&2
233235
return 2
234236
fi
@@ -283,7 +285,7 @@ _retry_file_curl_internal() {
283285
# checksToRun are conditions that need to pass to stop the retry loop. If not passed, eval command will return 0, because checksToRun will be interpreted as an empty string.
284286
retries=$1; waitSleep=$2; timeout=$3; filePath=$4; url=$5; checksToRun=( "${@:6}" )
285287
echo "${retries} file curl retries"
286-
for i in $(seq 1 $retries); do
288+
for i in $(seq 1 $retries); do
287289
# Use eval to execute the checksToRun string as a command
288290
( eval "$checksToRun" ) && break || if [ "$i" -eq "$retries" ]; then
289291
return 1
@@ -456,7 +458,7 @@ systemctlEnableAndStart() {
456458
fi
457459
}
458460

459-
systemctlEnableAndStartNoBlock() {
461+
systemctlEnableAndStartNoBlock() {
460462
service=$1; timeout=$2; status_check_delay_seconds=${3:-"0"}
461463

462464
systemctl_restart_no_block 100 5 $timeout $service
@@ -501,15 +503,15 @@ systemctlDisableAndStop() {
501503
semverCompare() {
502504
VERSION_A=$(echo $1 | cut -d "+" -f 1 | cut -d "~" -f 1)
503505
VERSION_B=$(echo $2 | cut -d "+" -f 1 | cut -d "~" -f 1)
504-
506+
505507
[ "${VERSION_A}" = "${VERSION_B}" ] && return 0
506508
sorted=$(echo ${VERSION_A} ${VERSION_B} | tr ' ' '\n' | sort -V )
507509
highestVersion=$(IFS= echo "${sorted}" | cut -d$'\n' -f2)
508510
[ "${VERSION_A}" = ${highestVersion} ] && return 0
509511
return 1
510512
}
511513

512-
514+
513515

514516
apt_get_download() {
515517
retries=$1; wait_sleep=$2; shift && shift;
@@ -577,7 +579,7 @@ logs_to_events() {
577579
--arg EventTid "0" \
578580
'{Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid}'
579581
)
580-
582+
581583
mkdir -p ${EVENTS_LOGGING_DIR}
582584
echo ${json_string} > ${EVENTS_LOGGING_DIR}${eventsFileName}.json
583585

@@ -781,7 +783,7 @@ updateMultiArchVersions() {
781783
# check if multiArchVersions not exists
782784
if [ "$(echo "${imageToBePulled}" | jq -r '.multiArchVersions | if . == null then "null" else empty end')" = "null" ]; then
783785
MULTI_ARCH_VERSIONS=()
784-
return
786+
return
785787
fi
786788

787789
local versions=($(echo "${imageToBePulled}" | jq -r ".multiArchVersions[]"))
@@ -818,7 +820,7 @@ getLatestPkgVersionFromK8sVersion() {
818820
local os_version="$4"
819821

820822
k8sMajorMinorVersion="$(echo "$k8sVersion" | cut -d- -f1 | cut -d. -f1,2)"
821-
823+
822824
package=$(jq ".Packages" "$COMPONENTS_FILEPATH" | jq ".[] | select(.name == \"${componentName}\")")
823825
PACKAGE_VERSIONS=()
824826
updatePackageVersions "${package}" "${os}" "${os_version}"
@@ -1043,12 +1045,12 @@ oras_login_with_kubelet_identity() {
10431045
configureSSHService() {
10441046
local os_param="${1:-$OS}"
10451047
local os_version_param="${2:-$OS_VERSION}"
1046-
1048+
10471049
# If not Ubuntu, no changes needed
10481050
if [ "$os_param" != "$UBUNTU_OS_NAME" ]; then
10491051
return 0
10501052
fi
1051-
1053+
10521054
# Only for Ubuntu 22.10+ or newer socket activation is used, for earlier versions no changes needed
10531055
if semverCompare "22.10" "$os_version_param" ; then
10541056
return 0
@@ -1065,11 +1067,11 @@ configureSSHService() {
10651067
if [ -f /etc/systemd/system/ssh.service.d/00-socket.conf ]; then
10661068
rm /etc/systemd/system/ssh.service.d/00-socket.conf || echo "Warning: Could not remove 00-socket.conf"
10671069
fi
1068-
1070+
10691071
if [ -f /etc/systemd/system/ssh.socket.d/addresses.conf ]; then
10701072
rm /etc/systemd/system/ssh.socket.d/addresses.conf || echo "Warning: Could not remove addresses.conf"
10711073
fi
1072-
1074+
10731075
# For all Ubuntu versions, just make sure ssh service is enabled and running
10741076
if ! systemctl is-enabled --quiet ssh.service; then
10751077
echo "Enabling SSH service..."
@@ -1080,7 +1082,7 @@ configureSSHService() {
10801082
echo "Error: Failed to start SSH service after configuration changes"
10811083
return $ERR_SYSTEMCTL_START_FAIL
10821084
fi
1083-
1085+
10841086
echo "SSH service successfully reconfigured and started"
10851087
return 0
10861088
}
@@ -1103,4 +1105,4 @@ extract_tarball() {
11031105
esac
11041106
}
11051107

1106-
#HELPERSEOF
1108+
#HELPERSEOF

0 commit comments

Comments
 (0)