[Build] Install the NVIDIA driver and CUDA toolkit from the distribution package manager using NVIDIA local repo packages instead of the run file installers.

Author: gmarciani

CHANGELOG.md

@@ -14,6 +14,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 
 **CHANGES**
+- Install the NVIDIA driver and CUDA toolkit from the distribution package manager using NVIDIA local repo packages instead of the run file installers.
 - In GPU Health Check, skip DCGM diagnostics when NVIDIA MIG is enabled because dcgmi diag does not support MIG.
 
 **CHANGES**

cookbooks/aws-parallelcluster-platform/kitchen.platform-install.yml

@@ -110,6 +110,7 @@ suites:
         - resource:package_repos
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
+        - resource:nvidia_repo:add
       cluster:
         nvidia:
           enabled: true
@@ -142,8 +143,9 @@ suites:
         - resource:package_repos
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
+        - resource:nvidia_repo:add
         - resource:nvidia_driver
-        - recipe:aws-parallelcluster-platform::cuda
+        - resource:nvidia_cuda
       cluster:
         nvidia:
           enabled: true
@@ -263,18 +265,20 @@ suites:
   - name: nvidia_cuda
     run_list:
       - recipe[aws-parallelcluster-tests::setup]
-      - recipe[aws-parallelcluster-platform::cuda]
+      - recipe[aws-parallelcluster-tests::test_resource]
     verifier:
       controls:
         - /tag:install_.*cuda/
     driver:
       # nvidia_driver can be executed only on a graphic EC2 instance example: g5.xlarge(x86_86) or g5g.xlarge(aarm64)
       instance_type: g4dn.2xlarge
     attributes:
+      resource: nvidia_cuda
       dependencies:
         - resource:package_repos
         - 'resource:package { "package_name": "dkms" }'
         - resource:build_tools
+        - resource:nvidia_repo:add
       cluster:
         nvidia:
           enabled: true

cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-#
-# Cookbook:: aws-parallelcluster
-# Recipe:: cuda
-#
-# Copyright:: 2013-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
-# License. A copy of the License is located at
-#
-# http://aws.amazon.com/apache2.0/
-#
-# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
-# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
-# limitations under the License.
-
-return unless nvidia_enabled?
-
-# Cuda installer from https://developer.nvidia.com/cuda-toolkit-archive
-# Cuda installer naming: cuda_11.8.0_520.61.05_linux
-
-cuda_full_version = node['cluster']['nvidia']['cuda']['version']
-cuda_parts = cuda_full_version.split('.')
-cuda_version = "#{cuda_parts[0]}.#{cuda_parts[1]}"
-cuda_version_suffix = node['cluster']['nvidia']['cuda']['driver_version_suffix']
-cuda_samples_version = cuda_version
-cuda_arch = arm_instance? ? 'linux_sbsa' : 'linux'
-cuda_url = "#{node['cluster']['nvidia']['cuda']['base_url']}/cuda_#{cuda_full_version}_#{cuda_version_suffix}_#{cuda_arch}.run"
-cuda_samples_url = "#{node['cluster']['nvidia']['cuda']['samples_base_url']}/v#{cuda_samples_version}.tar.gz"
-tmp_cuda_run = '/tmp/cuda.run'
-tmp_cuda_sample_archive = '/tmp/cuda-sample.tar.gz'
-
-# Save resolved CUDA versions to .default so InSpec/build-image tests which run during ParallelClusterTestComponent see them (mirrors Nvidia driver).
-node.default['cluster']['nvidia']['cuda']['version'] = cuda_full_version
-node.default['cluster']['nvidia']['cuda_samples_version'] = cuda_samples_version
-node_attributes 'Save cuda and cuda samples versions for InSpec tests'
-
-# Get CUDA run file
-remote_file tmp_cuda_run do
-  source cuda_url
-  mode '0755'
-  retries 3
-  retry_delay 5
-  not_if { ::File.exist?("/usr/local/cuda-#{cuda_version}") }
-end
-
-# Install CUDA driver
-bash 'cuda.run advanced' do
-  user 'root'
-  group 'root'
-  cwd '/tmp'
-  code <<-CUDA
-    set -e
-    mkdir /cuda-install
-    ./cuda.run --silent --toolkit --samples --tmpdir=/cuda-install
-    rm -rf /cuda-install
-    rm -f /tmp/cuda.run
-  CUDA
-  creates "/usr/local/cuda-#{cuda_version}"
-end
-
-# Get CUDA Sample Files
-remote_file tmp_cuda_sample_archive do
-  source cuda_samples_url
-  mode '0644'
-  retries 3
-  retry_delay 5
-  not_if { ::File.exist?("/usr/local/cuda-#{cuda_version}/samples") }
-end
-
-# Unpack CUDA Samples
-bash 'cuda.sample install' do
-  user 'root'
-  group 'root'
-  cwd '/tmp'
-  code <<-CUDA
-    set -e
-    tar xf "/tmp/cuda-sample.tar.gz" --directory "/usr/local/"
-    rm -f "/tmp/cuda-sample.tar.gz"
-  CUDA
-  creates "/usr/local/cuda-#{cuda_version}/samples"
-end

cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb

@@ -15,9 +15,14 @@
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Register the NVIDIA local repos (driver and CUDA) up front so that the
+# nvidia_driver and nvidia_cuda resources can install their packages from them.
+# Versions and URLs default to node attributes and can be overridden there.
+nvidia_repo 'Install NVIDIA local repos'
+
 nvidia_driver 'Install Nvidia driver'
 
-include_recipe "aws-parallelcluster-platform::cuda"
+nvidia_cuda 'Install Nvidia CUDA'
 
 gdrcopy 'Install Nvidia gdrcopy'
 
@@ -28,3 +33,8 @@
 nvidia_dcgm 'install Nvidia datacenter-gpu-manager'
 
 nvidia_imex 'Install nvidia-imex'
+
+# Remove the NVIDIA local repos now that all NVIDIA packages have been installed
+nvidia_repo 'Remove NVIDIA local repos' do
+  action :remove
+end

cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb

@@ -21,12 +21,17 @@
 action :setup do
   return unless _fabric_manager_enabled
 
-  # Share fabric manager package and version with InSpec tests
+  already_installed = fabric_manager_installed?
+
+  # Always expose the package name to InSpec. Record the configured version only
+  # when pcluster installs it, so a Fabric Manager shipped by the base image
+  # (e.g. DLAMI) is only checked for presence, not for pcluster's version, and is
+  # left untouched.
   node.default['cluster']['nvidia']['fabricmanager']['package'] = fabric_manager_package
-  node.default['cluster']['nvidia']['fabricmanager']['version'] = fabric_manager_version
+  node.default['cluster']['nvidia']['fabricmanager']['version'] = fabric_manager_version unless already_installed
   node_attributes "dump node attributes"
 
-  action_install_package
+  action_install_package unless already_installed
 end
 
 action :configure do
@@ -40,8 +45,15 @@
 end
 
 def _fabric_manager_enabled
-  # NVIDIA Fabric Manager not present on ARM
-  !arm_instance? && _nvidia_enabled
+  # NVIDIA Fabric Manager not present on ARM, and it requires the NVIDIA driver.
+  # Install only when NVIDIA is enabled and the driver is already installed.
+  !arm_instance? && _nvidia_enabled && nvidia_installed?
+end
+
+# True if Fabric Manager is already installed (e.g. shipped by the base image
+# such as the DLAMI). nv-fabricmanager is installed to /usr/bin on all platforms.
+def fabric_manager_installed?
+  ::File.exist?('/usr/bin/nv-fabricmanager')
 end
 
 def _nvidia_enabled

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_alinux2023.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'amazon' do |node|
+  node['platform_version'].to_i == 2023
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_rhel.rb'

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_redhat8.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'redhat' do |node|
+  node['platform_version'].to_i >= 8
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_rhel.rb'

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_rocky8.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'rocky' do |node|
+  node['platform_version'].to_i >= 8
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_rhel.rb'

cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_ubuntu22+.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_cuda, platform: 'ubuntu' do |node|
+  node['platform_version'].to_i >= 22
+end
+
+use 'partial/_nvidia_cuda_common.rb'
+use 'partial/_nvidia_cuda_install_debian.rb'