[Build] Install the NVIDIA driver and CUDA toolkit from the distribution package manager using NVIDIA local repo packages instead of the run file installers.

Author: gmarciani

CHANGELOG.md

@@ -14,6 +14,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 
 **CHANGES**
+- Install the NVIDIA driver and CUDA toolkit from the distribution package manager using NVIDIA local repo packages instead of the run file installers.
 - In GPU Health Check, skip DCGM diagnostics when NVIDIA MIG is enabled because dcgmi diag does not support MIG.
 
 **CHANGES**
@@ -30,7 +31,7 @@ via custom Slurm settings or the cluster name contains upper-case letters.
 - Fix compute node bootstrap hanging without a clear error when the compute subnet cannot reach DynamoDB.
 - Fix login nodes not mounting `/opt/parallelcluster/shared` when EFS is used as the internal shared storage type.
 - Fix SELinux not actually being disabled on RHEL-family OSes (kernels >= 6.4) due to a [deprecated mechanism](https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-runtime-disable) being silently ignored by newer kernels.
-- Fix `build-image` failure by skipping installation of `gdrcopy` and `dcgm` if parent image has an existing installed version.
+- Fix `build-image` failure by skipping installation of `fabric-manager`, `gdrcopy` and `dcgm` if parent image has an existing installed version.
 
 **DEPRECATIONS**
 - Amazon Linux 2 is no longer supported.

cookbooks/aws-parallelcluster-platform/kitchen.platform-install.yml

@@ -110,6 +110,7 @@ suites:
         - resource:package_repos
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
+        - resource:nvidia_repo:add
       cluster:
         nvidia:
           enabled: true
@@ -142,8 +143,9 @@ suites:
         - resource:package_repos
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
+        - resource:nvidia_repo:add
         - resource:nvidia_driver
-        - recipe:aws-parallelcluster-platform::cuda
+        - resource:nvidia_cuda
       cluster:
         nvidia:
           enabled: true
@@ -263,18 +265,20 @@ suites:
   - name: nvidia_cuda
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-platform::cuda]
+      - recipe[aws-parallelcluster-tests::test_resource]
     verifier:
       controls:
         - /tag:install_.*cuda/
     driver:
       # nvidia_driver can be executed only on a graphic EC2 instance example: g5.xlarge(x86_86) or g5g.xlarge(aarm64)
       instance_type: g4dn.2xlarge
     attributes:
+      resource: nvidia_cuda
       dependencies:
         - resource:package_repos
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
+        - resource:nvidia_repo:add
       cluster:
         nvidia:
           enabled: true

cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb

@@ -14,6 +14,15 @@ def nvidia_enabled?
   ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled'])
 end
 
+# Convert a full CUDA version to its 'major-minor' dashed form used in CUDA repo
+# and package names, e.g. '13-0' for '13.0.2'. The patch level (the '.2') is
+# intentionally dropped because CUDA repos and packages are keyed on major-minor
+# only (e.g. 'cuda-repo-rhel9-13-0-local', 'cuda-toolkit-13-0').
+def cuda_version_dash(cuda_version)
+  major, minor = cuda_version.split('.')
+  "#{major}-#{minor}"
+end
+
 # Whether a base_url attribute still points to the default ParallelCluster artifacts location.
 # Used by: fabric manager, dcgm, nvlsm, enroot (caps).
 # Limitation: this check assumes attribute precedence resolves at attribute-load time

cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-#
-# Cookbook:: aws-parallelcluster
-# Recipe:: cuda
-#
-# Copyright:: 2013-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
-# License. A copy of the License is located at
-#
-# http://aws.amazon.com/apache2.0/
-#
-# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
-# limitations under the License.
-
-return unless nvidia_enabled?
-
-# Cuda installer from https://developer.nvidia.com/cuda-toolkit-archive
-# Cuda installer naming: cuda_11.8.0_520.61.05_linux
-
-cuda_full_version = node['cluster']['nvidia']['cuda']['version']
-cuda_parts = cuda_full_version.split('.')
-cuda_version = "#{cuda_parts[0]}.#{cuda_parts[1]}"
-cuda_version_suffix = node['cluster']['nvidia']['cuda']['driver_version_suffix']
-cuda_samples_version = cuda_version
-cuda_arch = arm_instance? ? 'linux_sbsa' : 'linux'
-cuda_url = "#{node['cluster']['nvidia']['cuda']['base_url']}/cuda_#{cuda_full_version}_#{cuda_version_suffix}_#{cuda_arch}.run"
-cuda_samples_url = "#{node['cluster']['nvidia']['cuda']['samples_base_url']}/v#{cuda_samples_version}.tar.gz"
-tmp_cuda_run = '/tmp/cuda.run'
-tmp_cuda_sample_archive = '/tmp/cuda-sample.tar.gz'
-
-# Save resolved CUDA versions to .default so InSpec/build-image tests which run during ParallelClusterTestComponent see them (mirrors Nvidia driver).
-node.default['cluster']['nvidia']['cuda']['version'] = cuda_full_version
-node.default['cluster']['nvidia']['cuda_samples_version'] = cuda_samples_version
-node_attributes 'Save cuda and cuda samples versions for InSpec tests'
-
-# Get CUDA run file
-remote_file tmp_cuda_run do
-  source cuda_url
-  mode '0755'
-  retries 3
-  retry_delay 5
-  not_if { ::File.exist?("/usr/local/cuda-#{cuda_version}") }
-end
-
-# Install CUDA driver
-bash 'cuda.run advanced' do
-  user 'root'
-  group 'root'
-  cwd '/tmp'
-  code <<-CUDA
-    set -e
-    mkdir /cuda-install
-    ./cuda.run --silent --toolkit --samples --tmpdir=/cuda-install
-    rm -rf /cuda-install
-    rm -f /tmp/cuda.run
-  CUDA
-  creates "/usr/local/cuda-#{cuda_version}"
-end
-
-# Get CUDA Sample Files
-remote_file tmp_cuda_sample_archive do
-  source cuda_samples_url
-  mode '0644'
-  retries 3
-  retry_delay 5
-  not_if { ::File.exist?("/usr/local/cuda-#{cuda_version}/samples") }
-end
-
-# Unpack CUDA Samples
-bash 'cuda.sample install' do
-  user 'root'
-  group 'root'
-  cwd '/tmp'
-  code <<-CUDA
-    set -e
-    tar xf "/tmp/cuda-sample.tar.gz" --directory "/usr/local/"
-    rm -f "/tmp/cuda-sample.tar.gz"
-  CUDA
-  creates "/usr/local/cuda-#{cuda_version}/samples"
-end

cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb

@@ -15,9 +15,14 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Register the NVIDIA local repos (driver and CUDA) up front so that the
+# nvidia_driver and nvidia_cuda resources can install their packages from them.
+# Versions and URLs default to node attributes and can be overridden there.
+nvidia_repo 'Install NVIDIA local repos'
+
 nvidia_driver 'Install Nvidia driver'
 
-include_recipe "aws-parallelcluster-platform::cuda"
+nvidia_cuda 'Install Nvidia CUDA'
 
 gdrcopy 'Install Nvidia gdrcopy'
 
@@ -28,3 +33,8 @@
 nvidia_dcgm 'install Nvidia datacenter-gpu-manager'
 
 nvidia_imex 'Install nvidia-imex'
+
+# Remove the NVIDIA local repos now that all NVIDIA packages have been installed
+nvidia_repo 'Remove NVIDIA local repos' do
+  action :remove
+end

cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb

@@ -20,6 +20,7 @@
 
 action :setup do
   return unless _fabric_manager_enabled
+  return if fabric_manager_installed?
 
   # Share fabric manager package and version with InSpec tests
   node.default['cluster']['nvidia']['fabricmanager']['package'] = fabric_manager_package
@@ -44,6 +45,12 @@ def _fabric_manager_enabled
   !arm_instance? && _nvidia_enabled
 end
 
+# True if Fabric Manager is already installed (e.g. shipped by the base image
+# such as the DLAMI). nv-fabricmanager is installed to /usr/bin on all platforms.
+def fabric_manager_installed?
+  ::File.exist?('/usr/bin/nv-fabricmanager')
+end
+
 def _nvidia_enabled
   nvidia_enabled.nil? ? ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) : nvidia_enabled
 end

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_alinux2023.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'amazon' do |node|
+  node['platform_version'].to_i == 2023
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_rhel.rb'

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_redhat8+.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'redhat' do |node|
+  node['platform_version'].to_i >= 8
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_rhel.rb'

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_rocky8+.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'rocky' do |node|
+  node['platform_version'].to_i >= 8
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_rhel.rb'

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_ubuntu22+.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'ubuntu' do |node|
+  node['platform_version'].to_i >= 22
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_debian.rb'