diff --git a/CHANGELOG.md b/CHANGELOG.md index 14bf6ce10e..6318693c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste **CHANGES** - Enforce NFSv4-only on the ParallelCluster-managed NFS server (head node). The NFSv3 client stack (rpcbind, rpc-statd, lockd) are unchanged, so cluster nodes can still mount external NFSv3 servers. +- Install the NVIDIA driver and CUDA toolkit from the distribution package manager using NVIDIA local repo packages instead of the run file installers. - In GPU Health Check, skip DCGM diagnostics when NVIDIA MIG is enabled because dcgmi diag does not support MIG. - Upgrade Slurm to version 25.11.6 (from 25.11.4). - Upgrade EFA installer to 1.49.0 (from 1.47.0). @@ -37,7 +38,7 @@ via custom Slurm settings or the cluster name contains upper-case letters. - Fix compute node bootstrap hanging without a clear error when the compute subnet cannot reach DynamoDB. - Fix login nodes not mounting `/opt/parallelcluster/shared` when EFS is used as the internal shared storage type. - Fix SELinux not actually being disabled on RHEL-family OSes (kernels >= 6.4) due to a [deprecated mechanism](https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-runtime-disable) being silently ignored by newer kernels. -- Fix `build-image` failure by skipping installation of `gdrcopy` and `dcgm` if parent image has an existing installed version. +- Fix `build-image` failure by skipping installation of `fabric-manager`, `gdrcopy` and `dcgm` if parent image has an existing installed version. **DEPRECATIONS** - Amazon Linux 2 is no longer supported. diff --git a/cookbooks/aws-parallelcluster-platform/attributes/platform.rb b/cookbooks/aws-parallelcluster-platform/attributes/platform.rb index 10eca1c0f4..bdfd2fd18a 100644 --- a/cookbooks/aws-parallelcluster-platform/attributes/platform.rb +++ b/cookbooks/aws-parallelcluster-platform/attributes/platform.rb @@ -28,6 +28,11 @@ default['cluster']['nvidia']['driver_version'] = '580.105.08' default['cluster']['nvidia']['dcgm_version'] = '4.5.1-1' +# Extra packages installed from the NVIDIA local repo alongside the driver +# meta-package, as a comma-separated list. +# * nvidia-xconfig: required by DCV to generate the X configuration +default['cluster']['nvidia']['driver_extra_packages'] = 'nvidia-xconfig' + default['cluster']['nvidia']['driver_base_url'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_driver" default['cluster']['nvidia']['dcgm_base_url'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_dcgm" default['cluster']['nvidia']['fabricmanager']['base_url'] = "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_fabric" diff --git a/cookbooks/aws-parallelcluster-platform/kitchen.platform-install.yml b/cookbooks/aws-parallelcluster-platform/kitchen.platform-install.yml index 4ab7c59d72..58e658dbf8 100644 --- a/cookbooks/aws-parallelcluster-platform/kitchen.platform-install.yml +++ b/cookbooks/aws-parallelcluster-platform/kitchen.platform-install.yml @@ -110,6 +110,7 @@ suites: - resource:package_repos - 'resource:package { "package_name": "dkms" }' - resource:build_tools + - resource:nvidia_repo:add cluster: nvidia: enabled: true @@ -142,8 +143,9 @@ suites: - resource:package_repos - 'resource:package { "package_name": "dkms" }' - resource:build_tools + - resource:nvidia_repo:add - resource:nvidia_driver - - recipe:aws-parallelcluster-platform::cuda + - resource:nvidia_cuda cluster: nvidia: enabled: true @@ -263,7 +265,7 @@ suites: - name: nvidia_cuda run_list: - recipe[aws-parallelcluster-tests::setup] - - recipe[aws-parallelcluster-platform::cuda] + - recipe[aws-parallelcluster-tests::test_resource] verifier: controls: - /tag:install_.*cuda/ @@ -271,10 +273,12 @@ suites: # nvidia_driver can be executed only on a graphic EC2 instance example: g5.xlarge(x86_86) or g5g.xlarge(aarm64) instance_type: g4dn.2xlarge attributes: + resource: nvidia_cuda dependencies: - resource:package_repos - 'resource:package { "package_name": "dkms" }' - resource:build_tools + - resource:nvidia_repo:add cluster: nvidia: enabled: true diff --git a/cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb b/cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb index ff404acbc5..e40f72e2d8 100644 --- a/cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb +++ b/cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb @@ -14,6 +14,15 @@ def nvidia_enabled? ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) end +# Convert a full CUDA version to its 'major-minor' dashed form used in CUDA repo +# and package names, e.g. '13-0' for '13.0.2'. The patch level (the '.2') is +# intentionally dropped because CUDA repos and packages are keyed on major-minor +# only (e.g. 'cuda-repo-rhel9-13-0-local', 'cuda-toolkit-13-0'). +def cuda_version_dash(cuda_version) + major, minor = cuda_version.split('.') + "#{major}-#{minor}" +end + # Whether a base_url attribute still points to the default ParallelCluster artifacts location. # Used by: fabric manager, dcgm, nvlsm, enroot (caps). # Limitation: this check assumes attribute precedence resolves at attribute-load time diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb index 0d5324739d..e69de29bb2 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cuda.rb @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -# -# Cookbook:: aws-parallelcluster -# Recipe:: cuda -# -# Copyright:: 2013-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the -# License. A copy of the License is located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and -# limitations under the License. - -return unless nvidia_enabled? - -# Cuda installer from https://developer.nvidia.com/cuda-toolkit-archive -# Cuda installer naming: cuda_11.8.0_520.61.05_linux - -cuda_full_version = node['cluster']['nvidia']['cuda']['version'] -cuda_parts = cuda_full_version.split('.') -cuda_version = "#{cuda_parts[0]}.#{cuda_parts[1]}" -cuda_version_suffix = node['cluster']['nvidia']['cuda']['driver_version_suffix'] -cuda_samples_version = cuda_version -cuda_arch = arm_instance? ? 'linux_sbsa' : 'linux' -cuda_url = "#{node['cluster']['nvidia']['cuda']['base_url']}/cuda_#{cuda_full_version}_#{cuda_version_suffix}_#{cuda_arch}.run" -cuda_samples_url = "#{node['cluster']['nvidia']['cuda']['samples_base_url']}/v#{cuda_samples_version}.tar.gz" -tmp_cuda_run = '/tmp/cuda.run' -tmp_cuda_sample_archive = '/tmp/cuda-sample.tar.gz' - -# Save resolved CUDA versions to .default so InSpec/build-image tests which run during ParallelClusterTestComponent see them (mirrors Nvidia driver). -node.default['cluster']['nvidia']['cuda']['version'] = cuda_full_version -node.default['cluster']['nvidia']['cuda_samples_version'] = cuda_samples_version -node_attributes 'Save cuda and cuda samples versions for InSpec tests' - -# Get CUDA run file -remote_file tmp_cuda_run do - source cuda_url - mode '0755' - retries 3 - retry_delay 5 - not_if { ::File.exist?("/usr/local/cuda-#{cuda_version}") } -end - -# Install CUDA driver -bash 'cuda.run advanced' do - user 'root' - group 'root' - cwd '/tmp' - code <<-CUDA - set -e - mkdir /cuda-install - ./cuda.run --silent --toolkit --samples --tmpdir=/cuda-install - rm -rf /cuda-install - rm -f /tmp/cuda.run - CUDA - creates "/usr/local/cuda-#{cuda_version}" -end - -# Get CUDA Sample Files -remote_file tmp_cuda_sample_archive do - source cuda_samples_url - mode '0644' - retries 3 - retry_delay 5 - not_if { ::File.exist?("/usr/local/cuda-#{cuda_version}/samples") } -end - -# Unpack CUDA Samples -bash 'cuda.sample install' do - user 'root' - group 'root' - cwd '/tmp' - code <<-CUDA - set -e - tar xf "/tmp/cuda-sample.tar.gz" --directory "/usr/local/" - rm -f "/tmp/cuda-sample.tar.gz" - CUDA - creates "/usr/local/cuda-#{cuda_version}/samples" -end diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb index e96c6bd65a..9f8b2982b2 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb @@ -15,9 +15,14 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +# Register the NVIDIA local repos (driver and CUDA) up front so that the +# nvidia_driver and nvidia_cuda resources can install their packages from them. +# Versions and URLs default to node attributes and can be overridden there. +nvidia_repo 'Install NVIDIA local repos' + nvidia_driver 'Install Nvidia driver' -include_recipe "aws-parallelcluster-platform::cuda" +nvidia_cuda 'Install Nvidia CUDA' gdrcopy 'Install Nvidia gdrcopy' @@ -28,3 +33,8 @@ nvidia_dcgm 'install Nvidia datacenter-gpu-manager' nvidia_imex 'Install nvidia-imex' + +# Remove the NVIDIA local repos now that all NVIDIA packages have been installed +nvidia_repo 'Remove NVIDIA local repos' do + action :remove +end diff --git a/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb b/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb index d65bb2fbc6..a8fbcf62ce 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/fabric_manager/partial/_fabric_manager_common.rb @@ -20,6 +20,7 @@ action :setup do return unless _fabric_manager_enabled + return if fabric_manager_installed? # Share fabric manager package and version with InSpec tests node.default['cluster']['nvidia']['fabricmanager']['package'] = fabric_manager_package @@ -44,6 +45,12 @@ def _fabric_manager_enabled !arm_instance? && _nvidia_enabled end +# True if Fabric Manager is already installed (e.g. shipped by the base image +# such as the DLAMI). nv-fabricmanager is installed to /usr/bin on all platforms. +def fabric_manager_installed? + ::File.exist?('/usr/bin/nv-fabricmanager') +end + def _nvidia_enabled nvidia_enabled.nil? ? ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) : nvidia_enabled end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_alinux2023.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_alinux2023.rb new file mode 100644 index 0000000000..1d5ef196a9 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_alinux2023.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +provides :nvidia_cuda, platform: 'amazon' do |node| + node['platform_version'].to_i == 2023 +end + +use 'partial/_nvidia_cuda_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_redhat8+.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_redhat8+.rb new file mode 100644 index 0000000000..30e4d4aa72 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_redhat8+.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +provides :nvidia_cuda, platform: 'redhat' do |node| + node['platform_version'].to_i >= 8 +end + +use 'partial/_nvidia_cuda_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_rocky8+.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_rocky8+.rb new file mode 100644 index 0000000000..d13aada406 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_rocky8+.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +provides :nvidia_cuda, platform: 'rocky' do |node| + node['platform_version'].to_i >= 8 +end + +use 'partial/_nvidia_cuda_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_ubuntu22+.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_ubuntu22+.rb new file mode 100644 index 0000000000..20e40d6762 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/nvidia_cuda_ubuntu22+.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +provides :nvidia_cuda, platform: 'ubuntu' do |node| + node['platform_version'].to_i >= 22 +end + +use 'partial/_nvidia_cuda_common.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/partial/_nvidia_cuda_common.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/partial/_nvidia_cuda_common.rb new file mode 100644 index 0000000000..dec6d9b3ab --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_cuda/partial/_nvidia_cuda_common.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true +# +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +unified_mode true +default_action :setup + +# Full CUDA toolkit version, e.g. '13.0.2' +property :cuda_version, String, default: node['cluster']['nvidia']['cuda']['version'] + +action :setup do + return unless nvidia_enabled? + return if on_docker? + + # Skip the entire CUDA setup when any CUDA version is already installed + # (cuda_path is the version-agnostic symlink created by the toolkit). + return if ::File.exist?(cuda_path) + + # Share CUDA versions with InSpec tests. We expose the major.minor under a + # dedicated attribute and leave the canonical 'version' attribute as the full + # version, so other resources (e.g. nvidia_repo) can rely on it regardless of order. + node.default['cluster']['nvidia']['cuda']['major_minor_version'] = cuda_major_minor + node.default['cluster']['nvidia']['cuda_samples_version'] = cuda_major_minor + node_attributes 'Save cuda and cuda samples versions for InSpec tests' + + # Install the CUDA toolkit from the local repo. The built-in `package` resource + # dispatches to the platform's package manager (dnf on RHEL/Amazon Linux, apt on + # Ubuntu), so no platform-specific partial is needed. The CUDA local repo, and its + # refreshed package-manager metadata, is registered earlier in the nvidia install recipe. + package cuda_toolkit_package do + retries 3 + retry_delay 5 + end + + # Expose CUDA binaries and libraries to all users + template '/etc/profile.d/cuda.sh' do + source 'nvidia/cuda.sh.erb' + cookbook 'aws-parallelcluster-platform' + owner 'root' + group 'root' + mode '0644' + variables(cuda_path: cuda_path) + end + + # Download and unpack the CUDA samples. + remote_file cuda_samples_archive do + source cuda_samples_url + mode '0644' + retries 3 + retry_delay 5 + not_if { ::File.exist?(cuda_samples_dir) } + end + + bash 'cuda.sample install' do + user 'root' + group 'root' + cwd '/tmp' + code <<-CUDA + set -e + tar xf "#{cuda_samples_archive}" --directory "#{cuda_installation_base_dir}/" + rm -f "#{cuda_samples_archive}" + CUDA + creates cuda_samples_dir + end +end + +# CUDA 'major.minor', e.g. '13.0' for '13.0.2' +def cuda_major_minor + major, minor = cuda_version.split('.') + "#{major}.#{minor}" +end + +def cuda_toolkit_package + "cuda-toolkit-#{cuda_version_dash(cuda_version)}" +end + +# The CUDA toolkit package installs into a version-specific directory +# (e.g. /usr/local/cuda-13.0) and also creates a version-agnostic '/usr/local/cuda' +# symlink pointing to it. We use the two deliberately for different purposes: +# - cuda_install_dir: the versioned directory; used to locate the CUDA samples dir. +# - cuda_path: the stable symlink; used to detect whether any CUDA version is already +# installed (to skip installation) and injected into /etc/profile.d/cuda.sh so that +# users' PATH/LD_LIBRARY_PATH keep working across CUDA version changes. + +# Version-specific CUDA installation directory, e.g. '/usr/local/cuda-13.0' +def cuda_install_dir + "#{cuda_installation_base_dir}/cuda-#{cuda_major_minor}" +end + +# Directory where the CUDA samples are unpacked, e.g. '/usr/local/cuda-13.0/samples' +def cuda_samples_dir + "#{cuda_install_dir}/samples" +end + +# Version-agnostic CUDA directory, a symlink to the version-specific installation +# directory (cuda_install_dir), e.g. '/usr/local/cuda' -> '/usr/local/cuda-13.0'. +# Injected into /etc/profile.d/cuda.sh. +def cuda_path + "#{cuda_installation_base_dir}/cuda" +end + +# Base directory under which all CUDA software is installed +def cuda_installation_base_dir + '/usr/local' +end + +def cuda_samples_url + "#{node['cluster']['nvidia']['cuda']['samples_base_url']}/v#{cuda_major_minor}.tar.gz" +end + +def cuda_samples_archive + '/tmp/cuda-sample.tar.gz' +end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_alinux2023.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_alinux2023.rb index 66469ba93b..9661ddd7a2 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_alinux2023.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_alinux2023.rb @@ -17,6 +17,7 @@ end use 'partial/_nvidia_driver_common.rb' +use 'partial/_nvidia_driver_install_rhel.rb' def set_compiler? # Amazon linux 2 with Kernel 5 need to set CC to /usr/bin/gcc10-gcc using dkms override diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb index 5866b8ebcf..b97d00baaf 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_redhat8.rb @@ -17,3 +17,4 @@ end use 'partial/_nvidia_driver_common.rb' +use 'partial/_nvidia_driver_install_rhel.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb index 6c91e05936..a6fc9baae7 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_rocky8.rb @@ -17,3 +17,4 @@ end use 'partial/_nvidia_driver_common.rb' +use 'partial/_nvidia_driver_install_rhel.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_ubuntu22+.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_ubuntu22+.rb index dd3b5d8872..9b6cc47d7f 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_ubuntu22+.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/nvidia_driver_ubuntu22+.rb @@ -17,6 +17,7 @@ end use 'partial/_nvidia_driver_common.rb' +use 'partial/_nvidia_driver_install_debian.rb' def rebuild_initramfs? true diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_common.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_common.rb index 0df858dac7..c7689dd174 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_common.rb @@ -15,26 +15,19 @@ unified_mode true default_action :setup -property :nvidia_driver_version, String - -tmp_nvidia_run = '/tmp/nvidia.run' +property :nvidia_driver_version, String, default: node['cluster']['nvidia']['driver_version'] +property :extra_driver_packages, String, default: node['cluster']['nvidia']['driver_extra_packages'] action :setup do return unless nvidia_driver_enabled? + return if nvidia_driver_installed? return if on_docker? - # Share nvidia driver version with InSpec tests - node.default['cluster']['nvidia']['driver_version'] = _nvidia_driver_version + # Record the configured version that pcluster is about to install so InSpec + # verifies the right version. + node.default['cluster']['nvidia']['driver_version'] = new_resource.nvidia_driver_version node_attributes "Save Nvidia driver version for Inspec tests" - remote_file tmp_nvidia_run do - source nvidia_driver_url - mode '0755' - retries 3 - retry_delay 5 - not_if { ::File.exist?('/usr/bin/nvidia-smi') } - end - # Make sure nouveau kernel module is unloaded, otherwise installation of NVIDIA driver fails kernel_module 'nouveau' do action :uninstall @@ -79,17 +72,21 @@ end end - # Install driver - bash 'nvidia.run advanced' do - user 'root' - group 'root' - cwd '/tmp' - code <<-NVIDIA - set -e - #{compiler_path} ./nvidia.run --silent --dkms --disable-nouveau -m=#{nvidia_kernel_module} - rm -f /tmp/nvidia.run - NVIDIA - creates '/usr/bin/nvidia-smi' + # Prepare the system for the driver meta-package install (platform-specific) + action_prepare_driver_install + + # Install the driver meta-package from the NVIDIA local repo. + package nvidia_driver_package do + retries 3 + retry_delay 5 + end + + # Install the extra driver packages from the NVIDIA local repo. + new_resource.extra_driver_packages.split(',').each do |pkg| + package pkg do + retries 3 + retry_delay 5 + end end execute 'initramfs to remove nouveau' do @@ -98,20 +95,26 @@ end if rebuild_initramfs? end -def _nvidia_driver_version - nvidia_driver_version || node['cluster']['nvidia']['driver_version'] +# Whether the open-source kernel modules flavor must be installed. +def nvidia_open_kernel_modules? + !['false', 'no', false].include?(node['cluster']['nvidia']['kernel_open']) end -def nvidia_driver_url - "#{node['cluster']['nvidia']['driver_base_url']}/NVIDIA-Linux-#{nvidia_arch}-#{_nvidia_driver_version}.run" +# Driver meta-package installed from the local repo. +# Open kernel modules -> 'nvidia-open', proprietary -> 'cuda-drivers'. +def nvidia_driver_package + nvidia_open_kernel_modules? ? 'nvidia-open' : 'cuda-drivers' end def nvidia_driver_enabled? nvidia_enabled? end -def nvidia_arch - arm_instance? ? 'aarch64' : 'x86_64' +# True if the NVIDIA driver is already installed (e.g. shipped by the base image +# such as the DLAMI). nvidia-smi is the canonical signal of a healthy driver and +# is installed to /usr/bin on all platforms. +def nvidia_driver_installed? + ::File.exist?('/usr/bin/nvidia-smi') end def rebuild_initramfs? @@ -126,14 +129,6 @@ def compiler_path "" end -def nvidia_kernel_module - if ['false', 'no', false].include?(node['cluster']['nvidia']['kernel_open']) - "kernel" - else - "kernel-open" - end -end - def kernel_modules_to_load %w(drm_client_lib) end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_install_debian.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_install_debian.rb new file mode 100644 index 0000000000..d64d4be187 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_install_debian.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true +# +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +# Prepare the system for the driver meta-package install. +# Lock the driver to the exact version using NVIDIA's official version-locking package, +# installed *before* the driver as NVIDIA recommends. This +# pin is required on Debian/Ubuntu specifically: `nvidia-open`/`cuda-drivers` are +# "latest driver" meta-packages, so without the pin apt would resolve the driver to +# the newest version available across all enabled apt sources rather than the version +# shipped by our local repo, mismatching Fabric Manager / IMEX and the version recorded +# for the InSpec tests. The pinning package ships a precompiled apt-preferences file so we don't +# hand-maintain one. +# See https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/latest/version-locking.html +action :prepare_driver_install do + apt_package "nvidia-driver-pinning-#{new_resource.nvidia_driver_version}" do + retries 3 + retry_delay 5 + end +end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_install_rhel.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_install_rhel.rb new file mode 100644 index 0000000000..95bca6c790 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_driver/partial/_nvidia_driver_install_rhel.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true +# +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +# Prepare the system for the driver meta-package install. +# Enable the requested module stream and refresh the cache, mirroring the NVIDIA RHEL +# installation guide. +# See https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/latest/red-hat-enterprise-linux.html +# We invoke `dnf module enable` directly rather than via the community `dnf_module` +# resource, because that resource silently no-ops on Amazon Linux 2023. +action :prepare_driver_install do + execute 'Enable NVIDIA driver module' do + command "dnf -y module enable nvidia-driver:#{nvidia_driver_module_stream} && dnf clean all" + retries 3 + retry_delay 5 + end +end + +# DKMS module stream enabled via `dnf module enable nvidia-driver:`. +# Open kernel modules -> 'open-dkms', proprietary -> 'latest-dkms'. +def nvidia_driver_module_stream + nvidia_open_kernel_modules? ? 'open-dkms' : 'latest-dkms' +end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb index 205d6c6fe7..60a616e4ac 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_imex/partial/_nvidia_imex_common.rb @@ -99,6 +99,7 @@ def imex_installed? ::File.exist?("/usr/bin/#{nvidia_imex_service}") || ::File.exist?("/usr/bin/#{nvidia_imex_service}-ctl") end +# Install IMEX when NVIDIA is enabled or already installed. def nvidia_enabled_or_installed? nvidia_enabled? || nvidia_installed? end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_alinux2023.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_alinux2023.rb index 8553b3a850..f4dfe77e17 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_alinux2023.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_alinux2023.rb @@ -17,11 +17,8 @@ end use 'partial/_nvidia_repo_common.rb' +use 'partial/_nvidia_repo_rhel.rb' -def platform - 'rhel9' -end - -def repository_key - 'D42D0685.pub' +def local_repo_platform + 'amzn2023' end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb index 524e968fb4..594ea7a6e8 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_redhat8.rb @@ -17,11 +17,4 @@ end use 'partial/_nvidia_repo_common.rb' - -def platform - "rhel#{node['platform_version'].to_i}" -end - -def repository_key - 'D42D0685.pub' -end +use 'partial/_nvidia_repo_rhel.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb index c23c4a7ef1..2b384dc6bf 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_rocky8.rb @@ -17,11 +17,4 @@ end use 'partial/_nvidia_repo_common.rb' - -def platform - "rhel#{node['platform_version'].to_i}" -end - -def repository_key - 'D42D0685.pub' -end +use 'partial/_nvidia_repo_rhel.rb' diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_ubuntu22+.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_ubuntu22+.rb index 788e19efb7..7bdc17a6db 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_ubuntu22+.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/nvidia_repo_ubuntu22+.rb @@ -17,11 +17,8 @@ end use 'partial/_nvidia_repo_common.rb' +use 'partial/_nvidia_repo_debian.rb' -def platform +def local_repo_platform "ubuntu#{node['platform_version'].delete('.')}" end - -def repository_key - '3bf863cc.pub' -end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_common.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_common.rb index 75339119a0..6203be88e4 100644 --- a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_common.rb +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_common.rb @@ -1,41 +1,129 @@ +# frozen_string_literal: true +# +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + unified_mode true -default_action :setup +default_action :add + +# node.run_state keys recording which local repos this run actually added, +# so :remove only cleans up what :add installed. +DRIVER_REPO_ADDED = 'nvidia_driver_repo_added' +CUDA_REPO_ADDED = 'nvidia_cuda_repo_added' -property :aws_region, String +# Only the driver and CUDA versions are configurable per call; everything else +# (repo URLs and the CUDA driver-version suffix) is read from node attributes. +property :driver_version, String, default: node['cluster']['nvidia']['driver_version'] +property :cuda_version, String, default: node['cluster']['nvidia']['cuda']['version'] action :add do - package_repos 'update package repos' do - action :update + return unless nvidia_enabled? + return if on_docker? + + # Register the NVIDIA driver local repo unless the driver is already installed + # (e.g. pre-baked AMI). Track it so :remove only cleans up what this run added. + unless ::File.exist?('/usr/bin/nvidia-smi') + remote_file driver_repo_package_path do + source driver_repo_source_url + mode '0644' + retries 3 + retry_delay 5 + end + action_install_driver_repo + node.run_state[DRIVER_REPO_ADDED] = true end - package_repos 'add nvidia-repo' do - action :add - repo_name "nvidia-repo" - baseurl repository_url - gpgkey "#{repository_url}/#{repository_key}" - disable_modularity true + # Register the CUDA local repo unless CUDA is already installed. + unless ::File.exist?('/usr/local/cuda') + remote_file cuda_repo_package_path do + source cuda_repo_source_url + mode '0644' + retries 3 + retry_delay 5 + end + action_install_cuda_repo + node.run_state[CUDA_REPO_ADDED] = true end + + # Refresh the package manager metadata once, after all local repos have been + # enrolled, so their package lists (and, on RHEL, the driver repo's DKMS + # module metadata) are visible to the later nvidia_driver / nvidia_cuda installs. + action_refresh_repo_cache if node.run_state[DRIVER_REPO_ADDED] || node.run_state[CUDA_REPO_ADDED] end action :remove do - package_repos 'remove nvidia-repo' do - action :remove - repo_name "nvidia-repo" + # Only remove each local repo (and its downloaded installer) if this run added it. + # After removal, refresh the package manager metadata so the removed repo's package + # list stops advertising NVIDIA packages at the repo's (lower) versions, which would + # make later installs (e.g. enroot) attempt unwanted downgrades and fail. + if node.run_state[DRIVER_REPO_ADDED] + package driver_repo_package_name do + action local_repo_remove_action + end + + file driver_repo_package_path do + action :delete + end + + action_refresh_metadata + end + + if node.run_state[CUDA_REPO_ADDED] + package cuda_repo_package_name do + action local_repo_remove_action + end + + file cuda_repo_package_path do + action :delete + end + + action_refresh_metadata end end -def _aws_region - aws_region || node['cluster']['region'] +# --------------------------------------------------------------------------- +# NVIDIA driver local repo +# --------------------------------------------------------------------------- + +# Installed name of the NVIDIA driver local-repo package (no revision/arch suffix). +def driver_repo_package_name + "nvidia-driver-local-repo-#{local_repo_platform}-#{driver_version}" +end + +# Local filesystem path of the downloaded driver local-repo installer. +def driver_repo_package_path + "#{node['cluster']['sources_dir']}/#{driver_repo_package_file}" +end + +# Download URL of the driver local-repo installer. +def driver_repo_source_url + "#{node['cluster']['nvidia']['driver_base_url']}/#{driver_repo_package_file}" end -def arch_suffix - arm_instance? ? 'sbsa' : 'x86_64' +# --------------------------------------------------------------------------- +# CUDA local repo +# --------------------------------------------------------------------------- + +# Installed name of the CUDA local-repo package (no revision/arch suffix). +def cuda_repo_package_name + "cuda-repo-#{local_repo_platform}-#{cuda_version_dash(cuda_version)}-local" end -def repo_domain - _aws_region.start_with?('cn-') ? 'cn' : 'com' +# Local filesystem path of the downloaded CUDA local-repo installer. +def cuda_repo_package_path + "#{node['cluster']['sources_dir']}/#{cuda_repo_package_file}" end -def repository_url - "https://developer.download.nvidia.#{repo_domain}/compute/cuda/repos/#{platform}/#{arch_suffix}" +# Download URL of the CUDA local-repo installer. +def cuda_repo_source_url + "#{node['cluster']['nvidia']['cuda']['base_url']}/#{cuda_repo_package_file}" end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_debian.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_debian.rb new file mode 100644 index 0000000000..f84b0ac900 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_debian.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true +# +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +# Install the NVIDIA driver local-repo deb (downloaded earlier) and enroll its signing key. +action :install_driver_repo do + dpkg_package driver_repo_package_name do + source driver_repo_package_path + retries 3 + retry_delay 5 + end + + execute "Install keyring for #{driver_repo_package_name}" do + command "cp /var/#{driver_repo_package_name}/nvidia-driver-*-keyring.gpg /usr/share/keyrings/" + end +end + +# Install the CUDA local-repo deb (downloaded earlier) and enroll its signing key. +action :install_cuda_repo do + dpkg_package cuda_repo_package_name do + source cuda_repo_package_path + retries 3 + retry_delay 5 + end + + execute "Install keyring for #{cuda_repo_package_name}" do + command "cp /var/#{cuda_repo_package_name}/cuda-*-keyring.gpg /usr/share/keyrings/" + end +end + +action :refresh_repo_cache do + apt_update 'Update apt cache for NVIDIA local repos' do + action :update + end +end + +# Refresh apt's metadata, e.g. after removing the local repos, so their package +# lists are no longer offered to apt. +action :refresh_metadata do + apt_update 'Refresh apt metadata after removing NVIDIA local repos' do + action :update + end +end + +# Purge (not just remove) the local-repo package so its apt source list +# (/etc/apt/sources.list.d/*.list, shipped as a conffile) is deleted too. +# A plain remove keeps that conffile pointing at the now-deleted /var/ +# directory, which makes the subsequent `apt-get update` fail. +def local_repo_remove_action + :purge +end + +def arch_suffix + arm_instance? ? 'arm64' : 'amd64' +end + +# NVIDIA driver local-repo installer deb file, e.g. +# nvidia-driver-local-repo-ubuntu2204-580.105.08_1.0-1_amd64.deb +def driver_repo_package_file + "#{driver_repo_package_name}_1.0-1_#{arch_suffix}.deb" +end + +# CUDA local-repo installer deb file, e.g. +# cuda-repo-ubuntu2204-13-0-local_13.0.2-580.95.05-1_amd64.deb +def cuda_repo_package_file + "#{cuda_repo_package_name}_#{cuda_version}-#{node['cluster']['nvidia']['cuda']['driver_version_suffix']}-1_#{arch_suffix}.deb" +end diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_rhel.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_rhel.rb new file mode 100644 index 0000000000..35e5623c5a --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/resources/nvidia_repo/partial/_nvidia_repo_rhel.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true +# +# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. +# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. +# See the License for the specific language governing permissions and limitations under the License. + +# Install the NVIDIA driver local-repo rpm (downloaded earlier). +action :install_driver_repo do + rpm_package driver_repo_package_name do + source driver_repo_package_path + retries 3 + retry_delay 5 + end +end + +# Install the CUDA local-repo rpm (downloaded earlier). +action :install_cuda_repo do + rpm_package cuda_repo_package_name do + source cuda_repo_package_path + retries 3 + retry_delay 5 + end +end + +action :refresh_repo_cache do + execute 'Refresh dnf metadata for NVIDIA local repos' do + command 'dnf clean all' + end + + dnf_package 'Update dnf cache for NVIDIA local repos' do + action :flush_cache + end +end + +# Refresh dnf's on-disk metadata, e.g. after removing the local repos, so their +# package lists are no longer offered to dnf. +action :refresh_metadata do + execute 'Refresh dnf metadata after removing NVIDIA local repos' do + command 'dnf clean all' + end +end + +# rpm has no purge semantics; a plain remove already deletes the repo's +# .repo file, so removing the local-repo package is sufficient. +def local_repo_remove_action + :remove +end + +def arch_suffix + arm_instance? ? 'aarch64' : 'x86_64' +end + +# Default to the equivalent RHEL local repo. Platforms that publish their own +# local repo (e.g. Amazon Linux 2023) override this in their platform resource. +def local_repo_platform + "rhel#{node['platform_version'].to_i}" +end + +# NVIDIA driver local-repo installer rpm file, e.g. +# nvidia-driver-local-repo-amzn2023-580.105.08-1.0-1.x86_64.rpm +def driver_repo_package_file + "#{driver_repo_package_name}-1.0-1.#{arch_suffix}.rpm" +end + +# CUDA local-repo installer rpm file, e.g. +# cuda-repo-amzn2023-13-0-local-13.0.2_580.95.05-1.x86_64.rpm +def cuda_repo_package_file + "#{cuda_repo_package_name}-#{cuda_version}_#{node['cluster']['nvidia']['cuda']['driver_version_suffix']}-1.#{arch_suffix}.rpm" +end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/libraries/nvidia_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/libraries/nvidia_spec.rb index 30e83c93e2..d5d6d331a9 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/libraries/nvidia_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/libraries/nvidia_spec.rb @@ -27,12 +27,12 @@ end if should_install - it 'runs nvidia driver install bash' do - is_expected.to run_bash('nvidia.run advanced') + it 'installs the nvidia driver' do + is_expected.to create_cookbook_file('blacklist-nouveau.conf') end else - it 'does not run nvidia driver install bash' do - is_expected.not_to run_bash('nvidia.run advanced') + it 'does not install the nvidia driver' do + is_expected.not_to create_cookbook_file('blacklist-nouveau.conf') end end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cuda_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cuda_spec.rb deleted file mode 100644 index 1c78ad765f..0000000000 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cuda_spec.rb +++ /dev/null @@ -1,124 +0,0 @@ -require 'spec_helper' - -describe 'aws-parallelcluster-platform::cuda' do - cached(:cuda_version) { '13.0' } - cached(:cuda_patch) { '2' } - cached(:cuda_complete_version) { "#{cuda_version}.#{cuda_patch}" } - cached(:cuda_version_suffix) { '580.95.05' } - - context 'when nvidia not enabled' do - cached(:chef_run) do - allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) - ChefSpec::Runner.new.converge(described_recipe) - end - - it 'does not install cuda' do - is_expected.not_to run_bash('cuda.run advanced') - end - end - - context 'when on arm' do - cached(:cuda_arch) { 'linux_sbsa' } - cached(:cuda_url) { "#{node['cluster']['artifacts_s3_url']}/dependencies/cuda/cuda_#{cuda_complete_version}_#{cuda_version_suffix}_#{cuda_arch}.run" } - cached(:cuda_samples_version) { '13.0' } - cached(:cuda_samples_url) { "#{node['cluster']['artifacts_s3_url']}/dependencies/cuda/samples/v#{cuda_samples_version}.tar.gz" } - - cached(:chef_run) do - allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) - allow(::File).to receive(:exist?).with("/usr/local/cuda-#{cuda_version}").and_return(false) - allow(::File).to receive(:exist?).with("/usr/local/cuda-#{cuda_version}/samples").and_return(false) - ChefSpec::Runner.new.converge(described_recipe) - end - cached(:node) { chef_run.node } - - it 'saves cuda and cuda samples version' do - expect(node['cluster']['nvidia']['cuda']['version']).to eq(cuda_complete_version) - expect(node['cluster']['nvidia']['cuda_samples_version']).to eq(cuda_samples_version) - is_expected.to write_node_attributes('Save cuda and cuda samples versions for InSpec tests') - end - - it 'downloads CUDA run file' do - is_expected.to create_remote_file('/tmp/cuda.run').with( - source: cuda_url, - mode: '0755', - retries: 3, - retry_delay: 5 - ) - end - - it 'installs CUDA driver' do - # Install CUDA driver - is_expected.to run_bash('cuda.run advanced') - .with( - user: 'root', - group: 'root', - cwd: '/tmp', - creates: "/usr/local/cuda-#{cuda_version}") - .with_code(%r{mkdir /cuda-install}) - .with_code(%r{./cuda.run --silent --toolkit --samples --tmpdir=/cuda-install}) - .with_code(%r{rm -rf /cuda-install}) - .with_code(%r{rm -f /tmp/cuda.run}) - end - - it 'downloads CUDA sample files' do - is_expected.to create_remote_file('/tmp/cuda-sample.tar.gz').with( - source: cuda_samples_url, - mode: '0644', - retries: 3, - retry_delay: 5 - ) - end - - it 'unpacks CUDA samples' do - is_expected.to run_bash('cuda.sample install') - .with( - user: 'root', - group: 'root', - cwd: '/tmp') - .with_code(%r{tar xf "/tmp/cuda-sample.tar.gz" --directory "/usr/local/"}) - .with_code(%r{rm -f "/tmp/cuda-sample.tar.gz"}) - end - end - - context 'when not on arm' do - cached(:cuda_arch) { 'linux' } - cached(:default_base_url) { "#{node['cluster']['artifacts_s3_url']}/dependencies/cuda" } - cached(:default_samples_base_url) { "#{node['cluster']['artifacts_s3_url']}/dependencies/cuda/samples" } - - { - 'default S3 base_url' => { base_url: nil, samples_base_url: nil }, - 'base_url overridden via ExtraChefAttributes' => { - base_url: 'https://fake-public.example.DOMAIN/cuda', - samples_base_url: 'https://fake-public.example.DOMAIN/cuda-samples', - }, - }.each do |scenario, urls| - context "when #{scenario}" do - cached(:expected_base_url) { urls[:base_url] || default_base_url } - cached(:expected_samples_base_url) { urls[:samples_base_url] || default_samples_base_url } - cached(:cuda_url) { "#{expected_base_url}/cuda_#{cuda_complete_version}_#{cuda_version_suffix}_#{cuda_arch}.run" } - cached(:cuda_samples_url) { "#{expected_samples_base_url}/v#{cuda_version}.tar.gz" } - - cached(:chef_run) do - allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) - allow(::File).to receive(:exist?).with("/usr/local/cuda-#{cuda_version}").and_return(false) - allow(::File).to receive(:exist?).with("/usr/local/cuda-#{cuda_version}/samples").and_return(false) - ChefSpec::Runner.new do |node| - node.override['cluster']['nvidia']['cuda']['base_url'] = urls[:base_url] if urls[:base_url] - node.override['cluster']['nvidia']['cuda']['samples_base_url'] = urls[:samples_base_url] if urls[:samples_base_url] - end.converge(described_recipe) - end - cached(:node) { chef_run.node } - - it 'downloads CUDA run file from the expected URL' do - is_expected.to create_remote_file('/tmp/cuda.run').with_source(cuda_url) - end - - it 'downloads CUDA samples from the expected URL' do - is_expected.to create_remote_file('/tmp/cuda-sample.tar.gz').with_source(cuda_samples_url) - end - end - end - end -end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/nvidia_install_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/nvidia_install_spec.rb index 116d53fef5..e2e688fd1e 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/nvidia_install_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/nvidia_install_spec.rb @@ -9,12 +9,16 @@ end cached(:node) { chef_run.node } + it 'registers the NVIDIA local repos up front' do + is_expected.to add_nvidia_repo('Install NVIDIA local repos') + end + it 'installs nvidia driver' do is_expected.to setup_nvidia_driver('Install Nvidia driver') end it 'installs cuda' do - is_expected.to include_recipe('aws-parallelcluster-platform::cuda') + is_expected.to setup_nvidia_cuda('Install Nvidia CUDA') end it 'installs gdrcopy' do @@ -36,6 +40,10 @@ it 'installs nvidia_imex' do is_expected.to install_nvidia_imex('Install nvidia-imex') end + + it 'removes the NVIDIA local repos as the last step' do + is_expected.to remove_nvidia_repo('Remove NVIDIA local repos') + end end end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/fabric_manager_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/fabric_manager_spec.rb index 1c9d526715..027511bb7f 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/fabric_manager_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/fabric_manager_spec.rb @@ -175,6 +175,7 @@ def self.configure(chef_run) cached(:chef_run) do stubs_for_resource('fabric_manager') do |res| allow(res).to receive(:_fabric_manager_enabled).and_return(true) + allow(res).to receive(:fabric_manager_installed?).and_return(false) end runner = runner(platform: platform, version: version, step_into: ['fabric_manager']) ConvergeFabricManager.setup(runner, nvidia_driver_version: nvidia_driver_version) @@ -212,6 +213,23 @@ def self.configure(chef_run) end end end + + context 'when fabric manager is already installed (e.g. DLAMI)' do + cached(:chef_run) do + stubs_for_resource('fabric_manager') do |res| + allow(res).to receive(:_fabric_manager_enabled).and_return(true) + allow(res).to receive(:fabric_manager_installed?).and_return(true) + end + runner = runner(platform: platform, version: version, step_into: ['fabric_manager']) + ConvergeFabricManager.setup(runner, nvidia_driver_version: nvidia_driver_version) + end + cached(:node) { chef_run.node } + + it 'skips the install when fabric manager is already present' do + is_expected.not_to run_bash('install_fabricmanager_for_ubuntu') + is_expected.not_to run_bash("Install #{fabric_manager_package}") + end + end end end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_cuda_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_cuda_spec.rb new file mode 100644 index 0000000000..d848eb933e --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_cuda_spec.rb @@ -0,0 +1,150 @@ +require 'spec_helper' + +class ConvergeNvidiaCuda + def self.setup(chef_run, cuda_version: nil) + chef_run.converge_dsl('aws-parallelcluster-platform') do + nvidia_cuda 'setup' do + cuda_version cuda_version unless cuda_version.nil? + action :setup + end + end + end +end + +describe 'nvidia_cuda helpers' do + cached(:cuda_version) { '13.0.2' } + + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) + ChefSpec::SoloRunner.new(step_into: ['nvidia_cuda']) + end + cached(:resource) do + ConvergeNvidiaCuda.setup(chef_run, cuda_version: cuda_version) + chef_run.find_resource('nvidia_cuda', 'setup') + end + + it 'computes the cuda major.minor form' do + expect(resource.cuda_major_minor).to eq('13.0') + end + + it 'computes the cuda toolkit package name' do + expect(resource.cuda_toolkit_package).to eq('cuda-toolkit-13-0') + end + + it 'computes the install dir, samples dir and path' do + expect(resource.cuda_installation_base_dir).to eq('/usr/local') + expect(resource.cuda_install_dir).to eq('/usr/local/cuda-13.0') + expect(resource.cuda_samples_dir).to eq('/usr/local/cuda-13.0/samples') + expect(resource.cuda_path).to eq('/usr/local/cuda') + end + + it 'computes the samples archive and url' do + expect(resource.cuda_samples_archive).to eq('/tmp/cuda-sample.tar.gz') + expect(resource.cuda_samples_url).to eq("#{resource.node['cluster']['nvidia']['cuda']['samples_base_url']}/v13.0.tar.gz") + end +end + +describe 'nvidia_cuda: overriding samples_base_url to the official NVIDIA URL' do + cached(:cuda_version) { '13.0.2' } + cached(:samples_base_url) { 'https://github.com/NVIDIA/cuda-samples/archive/refs/tags' } + + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) + allow_any_instance_of(Object).to receive(:on_docker?).and_return(false) + mock_file_exists('/usr/local/cuda', false) + mock_file_exists('/usr/local/cuda-13.0/samples', false) + runner = runner(platform: 'amazon', version: '2023', step_into: ['nvidia_cuda']) do |node| + node.override['cluster']['nvidia']['cuda']['samples_base_url'] = samples_base_url + end + ConvergeNvidiaCuda.setup(runner, cuda_version: cuda_version) + end + + it 'downloads the cuda samples from the official NVIDIA URL' do + is_expected.to create_remote_file('/tmp/cuda-sample.tar.gz').with( + source: "#{samples_base_url}/v13.0.tar.gz" + ) + end +end + +describe 'nvidia_cuda:setup' do + for_all_oses do |platform, version| + cached(:cuda_version) { '13.0.2' } + + context "on #{platform}#{version} when nvidia not enabled" do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_cuda']) + ConvergeNvidiaCuda.setup(runner, cuda_version: cuda_version) + end + + it 'does not install the cuda toolkit' do + is_expected.not_to install_package('cuda-toolkit-13-0') + end + end + + context "on #{platform}#{version} when cuda is already installed" do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) + allow_any_instance_of(Object).to receive(:on_docker?).and_return(false) + mock_file_exists('/usr/local/cuda', true) + runner = runner(platform: platform, version: version, step_into: ['nvidia_cuda']) + ConvergeNvidiaCuda.setup(runner, cuda_version: cuda_version) + end + + it 'skips the entire cuda setup' do + is_expected.not_to install_package('cuda-toolkit-13-0') + is_expected.not_to create_remote_file('/tmp/cuda-sample.tar.gz') + is_expected.not_to create_template('/etc/profile.d/cuda.sh') + is_expected.not_to write_node_attributes('Save cuda and cuda samples versions for InSpec tests') + end + end + + context "on #{platform}#{version} when cuda is not yet installed" do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) + allow_any_instance_of(Object).to receive(:on_docker?).and_return(false) + mock_file_exists('/usr/local/cuda', false) + mock_file_exists('/usr/local/cuda-13.0/samples', false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_cuda']) + ConvergeNvidiaCuda.setup(runner, cuda_version: cuda_version) + end + cached(:node) { chef_run.node } + + it 'sets up nvidia_cuda' do + is_expected.to setup_nvidia_cuda('setup') + end + + it 'shares the cuda versions for InSpec' do + is_expected.to write_node_attributes('Save cuda and cuda samples versions for InSpec tests') + expect(node['cluster']['nvidia']['cuda']['major_minor_version']).to eq('13.0') + expect(node['cluster']['nvidia']['cuda_samples_version']).to eq('13.0') + end + + it 'leaves the canonical cuda version attribute untouched (full version)' do + expect(node['cluster']['nvidia']['cuda']['version']).to eq('13.0.2') + end + + it 'renders the cuda.sh profile with the cuda path' do + is_expected.to create_template('/etc/profile.d/cuda.sh').with( + source: 'nvidia/cuda.sh.erb', + cookbook: 'aws-parallelcluster-platform', + variables: { cuda_path: '/usr/local/cuda' } + ) + end + + it 'downloads and unpacks the cuda samples' do + is_expected.to create_remote_file('/tmp/cuda-sample.tar.gz') + is_expected.to run_bash('cuda.sample install').with( + creates: '/usr/local/cuda-13.0/samples' + ) + end + + it 'installs the cuda toolkit via the platform package manager' do + is_expected.to install_package('cuda-toolkit-13-0').with( + retries: 3, + retry_delay: 5 + ) + end + end + end +end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb index 9ab46cec1d..efa9ac4b5a 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_driver_spec.rb @@ -4,75 +4,65 @@ class ConvergeNvidiaDriver def self.setup(chef_run, nvidia_driver_version: nil) chef_run.converge_dsl('aws-parallelcluster-platform') do nvidia_driver 'setup' do - nvidia_driver_version nvidia_driver_version + nvidia_driver_version nvidia_driver_version unless nvidia_driver_version.nil? action :setup end end end end -describe 'nvidia_driver:_nvidia_driver_version' do +describe 'nvidia_driver:nvidia_driver_version' do cached(:nvidia_driver_attribute) { 'nvidia_driver_attribute' } cached(:nvidia_driver_property) { 'nvidia_driver_property' } - cached(:chef_run) do - ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) do |node| - node.override['cluster']['nvidia']['driver_version'] = nvidia_driver_attribute - end - end context 'when nvidia driver property is set' do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) + ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) do |node| + node.override['cluster']['nvidia']['driver_version'] = nvidia_driver_attribute + end + end cached(:resource) do ConvergeNvidiaDriver.setup(chef_run, nvidia_driver_version: nvidia_driver_property) chef_run.find_resource('nvidia_driver', 'setup') end it 'takes the value from nvidia driver property' do - expect(resource._nvidia_driver_version).to eq(nvidia_driver_property) + expect(resource.nvidia_driver_version).to eq(nvidia_driver_property) end end context 'when nvidia driver property is not set' do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) + ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) + end cached(:resource) do ConvergeNvidiaDriver.setup(chef_run) chef_run.find_resource('nvidia_driver', 'setup') end - it 'takes the value from nvidia driver attribute' do - expect(resource._nvidia_driver_version).to eq(nvidia_driver_attribute) + it 'defaults to the nvidia driver attribute' do + expect(resource.nvidia_driver_version).to eq(chef_run.node['cluster']['nvidia']['driver_version']) end end end -describe 'nvidia_driver:nvidia_driver_url' do - cached(:driver_version) { 'fake_driver_version' } - cached(:s3_base_url) { 'fake_s3_base_url/dependencies/nvidia_driver' } - cached(:public_base_url) { 'fake_public_nvidia_url/tesla' } - - { - 'default S3 base_url' => 'fake_s3_base_url/dependencies/nvidia_driver', - 'base_url overridden to public NVIDIA Tesla URL' => 'fake_public_nvidia_url/tesla', - }.each do |scenario, base_url| - context "when driver_base_url is #{scenario}" do +describe 'nvidia_driver:nvidia_open_kernel_modules?' do + [%w(false false), [false, false], %w(no false), %w(true true), [true, true], %w(yes true)].each do |kernel_open, expected_open| + context "node['cluster']['nvidia']['kernel_open'] is #{kernel_open}" do cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) do |node| - node.override['cluster']['nvidia']['driver_version'] = driver_version - node.override['cluster']['nvidia']['driver_base_url'] = base_url + node.override['cluster']['nvidia']['kernel_open'] = kernel_open end end - cached(:resource) do ConvergeNvidiaDriver.setup(chef_run) chef_run.find_resource('nvidia_driver', 'setup') end - - it 'constructs URL using x86_64 for non-ARM' do - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) - expect(resource.nvidia_driver_url).to eq("#{base_url}/NVIDIA-Linux-x86_64-#{driver_version}.run") - end - - it 'constructs URL using aarch64 for ARM' do - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) - expect(resource.nvidia_driver_url).to eq("#{base_url}/NVIDIA-Linux-aarch64-#{driver_version}.run") + it "is #{expected_open}" do + expect(resource.nvidia_open_kernel_modules?).to eq(expected_open == 'true' || expected_open == true) end end end @@ -90,37 +80,15 @@ def self.setup(chef_run, nvidia_driver_version: nil) end context "when nvidia not enabled" do - before do - allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) - end + before { allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) } it 'is false' do expect(resource.nvidia_driver_enabled?).to eq(false) end end - context "when nvidia enabled and arm instance" do - before do - allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) - end - - if platform == 'centos' - it 'is false' do - expect(resource.nvidia_driver_enabled?).to eq(false) - end - else - it 'is true' do - expect(resource.nvidia_driver_enabled?).to eq(true) - end - end - end - - context "when nvidia enabled and not arm instance" do - before do - allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) - end + context "when nvidia enabled" do + before { allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) } it 'is true' do expect(resource.nvidia_driver_enabled?).to eq(true) @@ -130,73 +98,10 @@ def self.setup(chef_run, nvidia_driver_version: nil) end end -describe 'nvidia_driver:nvidia_kernel_module' do - [%w(false kernel), [false, 'kernel'], %w(no kernel), %w(true kernel-open), [true, 'kernel-open'], %w(yes kernel-open)].each do |kernel_open, kernel_module| - context "node['cluster']['nvidia']['kernel_open'] is #{kernel_open}" do - cached(:chef_run) do - ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) do |node| - node.override['cluster']['nvidia']['kernel_open'] = kernel_open - end - end - cached(:resource) do - ConvergeNvidiaDriver.setup(chef_run) - chef_run.find_resource('nvidia_driver', 'setup') - end - it "is #{kernel_module}" do - allow_any_instance_of(Object).to receive(:nvidia_kernel_module).and_return(kernel_module) - expect(resource.nvidia_kernel_module).to eq(kernel_module) - end - end - end -end - -describe 'nvidia_driver:nvidia_arch' do - cached(:chef_run) do - ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) - end - - cached(:resource) do - ConvergeNvidiaDriver.setup(chef_run) - chef_run.find_resource('nvidia_driver', 'setup') - end - - context 'when on arm' do - it 'is aarch64' do - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) - expect(resource.nvidia_arch).to eq('aarch64') - end - end - - context 'when not on arm' do - it 'is x86_64' do - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) - expect(resource.nvidia_arch).to eq('x86_64') - end - end -end - -describe 'nvidia_driver:kernel_modules_to_load' do - cached(:chef_run) do - ChefSpec::SoloRunner.new(step_into: ['nvidia_driver']) - end - - cached(:resource) do - ConvergeNvidiaDriver.setup(chef_run) - chef_run.find_resource('nvidia_driver', 'setup') - end - - it 'returns expected kernel modules' do - expect(resource.kernel_modules_to_load).to eq(%w(drm_client_lib)) - end -end - describe 'nvidia_driver:setup' do for_all_oses do |platform, version| - cached(:nvidia_arch) { 'nvidia_arch' } - cached(:nvidia_kernel_module) { 'nvidia_kernel_module' } cached(:nvidia_driver_version) { 'nvidia_driver_version' } - cached(:kernel_modules_to_load) { %w(module1 module2) } - cached(:nvidia_driver_url) { "https://us.download.nvidia.com/tesla/#{nvidia_driver_version}/NVIDIA-Linux-#{nvidia_arch}-#{nvidia_driver_version}.run" } + cached(:debian?) { platform == 'ubuntu' } context "on #{platform}#{version} when nvidia_driver not enabled" do cached(:chef_run) do @@ -207,189 +112,140 @@ def self.setup(chef_run, nvidia_driver_version: nil) ConvergeNvidiaDriver.setup(runner) end - it 'does not install NVidia driver' do - is_expected.not_to run_bash('nvidia.run advanced') + it 'does not install the driver' do + is_expected.not_to run_execute('Enable NVIDIA driver module') + is_expected.not_to install_package('nvidia-open') end end - [%w(false kernel), %w(true kernel-open)].each do |kernel_open, kernel_module| - context "on #{platform}#{version} when nvidia_driver enabled and node['cluster']['nvidia']['kernel_open'] is #{kernel_open}" do - if platform == 'centos' - cached(:nvidia_driver_version) { '535.129.03' } - else - cached(:nvidia_driver_version) { 'nvidia_driver_version' } + context "on #{platform}#{version} when the driver is already installed" do + cached(:chef_run) do + stubs_for_resource('nvidia_driver') do |res| + allow(res).to receive(:nvidia_driver_enabled?).and_return(true) + allow(res).to receive(:kernel_modules_to_load).and_return([]) + allow(res).to receive(:gcc_major_version_used_by_kernel).and_return('12') + end + stub_command("lsinitramfs /boot/initrd.img-$(uname -r) | grep nouveau").and_return(false) + mock_file_exists('/usr/bin/nvidia-smi', true) + runner = runner(platform: platform, version: version, step_into: ['nvidia_driver']) do |node| + node.automatic['kernel']['release'] = '6.anything' end - cached(:nvidia_driver_url) { "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_driver/NVIDIA-Linux-#{nvidia_arch}-#{nvidia_driver_version}.run" } - cached(:kernel_compiler_version) { "KERNEL_COMPILER_VERSION" } - cached(:chef_run) do - stubs_for_resource('nvidia_driver') do |res| - allow(res).to receive(:nvidia_driver_enabled?).and_return(true) - allow(res).to receive(:nvidia_arch).and_return(nvidia_arch) - allow(res).to receive(:nvidia_kernel_module).and_return(kernel_module) - allow(res).to receive(:gcc_major_version_used_by_kernel).and_return(kernel_compiler_version) - allow(res).to receive(:kernel_modules_to_load).and_return(kernel_modules_to_load) - end + ConvergeNvidiaDriver.setup(runner, nvidia_driver_version: nvidia_driver_version) + end + cached(:node) { chef_run.node } - stub_command("lsinitramfs /boot/initrd.img-$(uname -r) | grep nouveau").and_return(true) - allow(::File).to receive(:exist?).with('/usr/bin/nvidia-smi').and_return(false) + it 'does not install the driver (nvidia-smi already present)' do + is_expected.not_to run_execute('Enable NVIDIA driver module') + is_expected.not_to install_package('nvidia-open') + end - runner = runner(platform: platform, version: version, step_into: ['nvidia_driver']) do |node| - node.automatic['kernel']['release'] = '5.anything' - end + it 'leaves the shipped driver untouched and records no version' do + is_expected.not_to write_node_attributes('Save Nvidia driver version for Inspec tests') + end + end - ConvergeNvidiaDriver.setup(runner, nvidia_driver_version: nvidia_driver_version) + context "on #{platform}#{version} when nvidia_driver enabled and not yet installed" do + cached(:chef_run) do + stubs_for_resource('nvidia_driver') do |res| + allow(res).to receive(:nvidia_driver_enabled?).and_return(true) + allow(res).to receive(:gcc_major_version_used_by_kernel).and_return('12') end - cached(:node) { chef_run.node } - - it 'dumps nodes attribues' do - is_expected.to write_node_attributes('Save Nvidia driver version for Inspec tests') + stub_command("lsinitramfs /boot/initrd.img-$(uname -r) | grep nouveau").and_return(true) + mock_file_exists('/usr/bin/nvidia-smi', false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_driver']) do |node| + node.automatic['kernel']['release'] = '6.anything' end + ConvergeNvidiaDriver.setup(runner, nvidia_driver_version: nvidia_driver_version) + end + cached(:node) { chef_run.node } - it 'sets up nvidia_driver' do - is_expected.to setup_nvidia_driver('setup') - end + it 'dumps node attributes for InSpec' do + is_expected.to write_node_attributes('Save Nvidia driver version for Inspec tests') + expect(node['cluster']['nvidia']['driver_version']).to eq(nvidia_driver_version) + end - it 'downloads nvidia driver' do - is_expected.to create_remote_file('/tmp/nvidia.run').with( - source: nvidia_driver_url, - mode: '0755', - retries: 3, - retry_delay: 5 - ) - end + it 'sets up nvidia_driver' do + is_expected.to setup_nvidia_driver('setup') + end - it 'uninstalls kernel module nouveau' do - is_expected.to uninstall_kernel_module('nouveau') - end + it 'uninstalls kernel module nouveau' do + is_expected.to uninstall_kernel_module('nouveau') + end - it 'creates file blacklist-nouveau.conf' do - is_expected.to create_cookbook_file('blacklist-nouveau.conf').with( - source: 'nvidia/blacklist-nouveau.conf', - path: '/etc/modprobe.d/blacklist-nouveau.conf', - owner: 'root', - group: 'root', - mode: '0644' - ) - end + it 'creates file blacklist-nouveau.conf' do + is_expected.to create_cookbook_file('blacklist-nouveau.conf').with( + source: 'nvidia/blacklist-nouveau.conf', + path: '/etc/modprobe.d/blacklist-nouveau.conf', + owner: 'root', + group: 'root', + mode: '0644' + ) + end - it 'loads kernel modules in they are exposed by the kernel' do - kernel_modules_to_load.each do |km| - is_expected.to run_execute("Load kernel module if exposed by the kernel: #{km}").with( - command: "if modinfo #{km}; then modprobe #{km}; fi" - ) - end + if platform == 'ubuntu' + it 'installs the open driver meta-package' do + is_expected.to install_package('nvidia-open') end - if platform == 'amazon' - compiler_version = 'gcc' - # Amazon Linux 2023 is expected to install the compiler and create nvidia conf when kernel version is 6. - # Here we are testing with kernel version 5 - it "does not install #{compiler_version}" do - is_expected.not_to install_package(compiler_version).with_retries(10).with_retry_delay(5) - end - - it 'does not create dkms/nvidia.conf' do - is_expected.not_to create_template('/etc/dkms/nvidia.conf') - end + it 'does not hold the driver package (version-locking package handles pinning)' do + is_expected.not_to run_execute('Hold nvidia-open') + end - it 'installs nvidia driver' do - is_expected.to run_bash('nvidia.run advanced') - .with( - user: 'root', - group: 'root', - cwd: '/tmp', - creates: '/usr/bin/nvidia-smi' - ) - .with_code(%r{CC=/usr/bin/gcc10-gcc ./nvidia.run --silent --dkms --disable-nouveau -m=#{kernel_module}}) - .with_code(%r{rm -f /tmp/nvidia.run}) - end - elsif platform == 'ubuntu' && version == '22.04' - it 'installs gcc' do - is_expected.to install_package('gcc').with_retries(10).with_retry_delay(5) - end + it 'installs the NVIDIA version-locking package for the requested version' do + is_expected.to install_apt_package("nvidia-driver-pinning-#{nvidia_driver_version}") + end - it 'creates dkms/nvidia.conf' do - compiler_path = "CC=/usr/bin/gcc-#{kernel_compiler_version}" - is_expected.to create_template('/etc/dkms/nvidia.conf').with( - source: 'nvidia/amazon/dkms/nvidia.conf.erb', - cookbook: 'aws-parallelcluster-platform', - owner: 'root', - group: 'root', - mode: '0644', - variables: { compiler_path: compiler_path } - ) - end - it 'installs nvidia driver' do - compiler_path = "CC=/usr/bin/gcc-#{kernel_compiler_version}" - is_expected.to run_bash('nvidia.run advanced') - .with( - user: 'root', - group: 'root', - cwd: '/tmp', - creates: '/usr/bin/nvidia-smi' - ) - .with_code(%r{#{compiler_path} ./nvidia.run --silent --dkms --disable-nouveau -m=#{kernel_module}}) - end - else - it "doesn't install gcc10" do - is_expected.not_to install_package('gcc10') - end - it 'installs nvidia driver' do - is_expected.to run_bash('nvidia.run advanced') - .with( - user: 'root', - group: 'root', - cwd: '/tmp', - creates: '/usr/bin/nvidia-smi' - ) - .with_code(%r{./nvidia.run --silent --dkms --disable-nouveau -m=#{kernel_module}}) - .with_code(%r{rm -f /tmp/nvidia.run}) - end + it 'rebuilds the initramfs' do + is_expected.to run_execute('initramfs to remove nouveau').with_command('update-initramfs -u') + end + else + it 'enables the open-dkms module stream and installs nvidia-open' do + is_expected.to run_execute('Enable NVIDIA driver module').with( + command: 'dnf -y module enable nvidia-driver:open-dkms && dnf clean all' + ) + is_expected.to install_package('nvidia-open') end - if platform == 'ubuntu' - it 'executes initramfs to remove nouveau' do - is_expected.to run_execute('initramfs to remove nouveau').with_command('update-initramfs -u') - end - else - it 'does not execute initramfs to remove nouveau' do - is_expected.not_to run_execute('initramfs to remove nouveau').with_command('update-initramfs -u') - end + it 'does not rebuild the initramfs' do + is_expected.not_to run_execute('initramfs to remove nouveau') end end end + end +end - context "on #{platform}#{version}" do +describe 'nvidia_driver:setup with proprietary kernel modules' do + for_all_oses do |platform, version| + cached(:nvidia_driver_version) { 'nvidia_driver_version' } + cached(:debian?) { platform == 'ubuntu' } + + context "on #{platform}#{version} when nvidia_driver enabled, proprietary and not yet installed" do cached(:chef_run) do stubs_for_resource('nvidia_driver') do |res| allow(res).to receive(:nvidia_driver_enabled?).and_return(true) - allow(res).to receive(:nvidia_arch).and_return(nvidia_arch) - allow(res).to receive(:nvidia_kernel_module).and_return(nvidia_kernel_module) - end - runner(platform: platform, version: version, step_into: ['nvidia_driver']) - end - cached(:node) { chef_run.node } - - context "when nouveau removed" do - before do - stub_command("lsinitramfs /boot/initrd.img-$(uname -r) | grep nouveau").and_return(false) - ConvergeNvidiaDriver.setup(chef_run, nvidia_driver_version: nvidia_driver_version) + allow(res).to receive(:gcc_major_version_used_by_kernel).and_return('12') end - - it 'does not execute initramfs to remove nouveau' do - is_expected.not_to run_execute('initramfs to remove nouveau').with_command('update-initramfs -u') + stub_command("lsinitramfs /boot/initrd.img-$(uname -r) | grep nouveau").and_return(true) + mock_file_exists('/usr/bin/nvidia-smi', false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_driver']) do |node| + node.automatic['kernel']['release'] = '6.anything' + node.override['cluster']['nvidia']['kernel_open'] = 'false' end + ConvergeNvidiaDriver.setup(runner, nvidia_driver_version: nvidia_driver_version) end - context "when kernel version is not 5" do - before do - stub_command("lsinitramfs /boot/initrd.img-$(uname -r) | grep nouveau").and_return(false) - node.automatic['kernel']['release'] = '4.anything' - ConvergeNvidiaDriver.setup(chef_run, nvidia_driver_version: nvidia_driver_version) + if platform == 'ubuntu' + it 'installs the proprietary cuda-drivers meta-package' do + is_expected.to install_package('cuda-drivers') + is_expected.not_to install_package('nvidia-open') end - - it "doesn't install gcc10" do - is_expected.not_to install_package('gcc10') + else + it 'enables the latest-dkms module stream and installs cuda-drivers' do + is_expected.to run_execute('Enable NVIDIA driver module').with( + command: 'dnf -y module enable nvidia-driver:latest-dkms && dnf clean all' + ) + is_expected.to install_package('cuda-drivers') + is_expected.not_to install_package('nvidia-open') end end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_imex_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_imex_spec.rb index a94bf15691..6224ac6eb6 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_imex_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_imex_spec.rb @@ -79,6 +79,17 @@ def self.configure(chef_run) expect(resource.nvidia_enabled_or_installed?).to eq(true) end end + + context "when nvidia is enabled and installed" do + before do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) + allow_any_instance_of(Object).to receive(:nvidia_installed?).and_return(true) + end + + it 'is true' do + expect(resource.nvidia_enabled_or_installed?).to eq(true) + end + end end end end @@ -181,6 +192,7 @@ def self.configure(chef_run) context 'when nvidia-imex binary already exists' do cached(:chef_run) do stubs_for_resource('nvidia_imex') do |res| + allow(res).to receive(:nvidia_enabled_or_installed?).and_return(true) allow(res).to receive(:imex_installed?).and_return(true) end runner = runner(platform: platform, version: version, step_into: ['nvidia_imex']) diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb index bb3a1b145f..5c6a8d4ab6 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_repo_spec.rb @@ -1,10 +1,11 @@ require 'spec_helper' class ConvergeNvidiaRepo - def self.add(chef_run, aws_region: 'any') + def self.add(chef_run, driver_version: nil, cuda_version: nil) chef_run.converge_dsl('aws-parallelcluster-platform') do nvidia_repo 'add' do - aws_region aws_region + driver_version driver_version unless driver_version.nil? + cuda_version cuda_version unless cuda_version.nil? action :add end end @@ -19,135 +20,409 @@ def self.remove(chef_run) end end -describe 'nvidia_repo:domain' do - cached(:chef_run) do - ChefSpec::SoloRunner.new(step_into: ['nvidia_repo']) +# Expected local-repo platform identifier for each supported OS. +def nvidia_local_repo_platform_for(platform, version) + case platform + when 'amazon' then 'amzn2023' + when 'ubuntu' then "ubuntu#{version.delete('.')}" + else "rhel#{version.to_i}" end +end - context 'when in China region' do - cached(:resource) do - ConvergeNvidiaRepo.add(chef_run, aws_region: 'cn-anything') - chef_run.find_resource('nvidia_repo', 'add') - end +describe 'nvidia_repo helpers' do + for_all_oses do |platform, version| + context "on #{platform}#{version}" do + cached(:driver_version) { '999.88.77' } + cached(:cuda_version) { '13.0.2' } + cached(:cuda_suffix) { '580.95.05' } + cached(:driver_base_url) { 'https://driver.example/nvidia_driver' } + cached(:cuda_base_url) { 'https://cuda.example/cuda' } + cached(:sources_dir) { '/fake/sources' } - it 'is cn' do - expect(resource.repo_domain).to eq('cn') + cached(:debian?) { platform == 'ubuntu' } + cached(:local_repo_platform) { nvidia_local_repo_platform_for(platform, version) } + cached(:arch) { debian? ? 'amd64' : 'x86_64' } + cached(:driver_pkg_name) { "nvidia-driver-local-repo-#{local_repo_platform}-#{driver_version}" } + cached(:cuda_pkg_name) { "cuda-repo-#{local_repo_platform}-13-0-local" } + cached(:driver_pkg_file) do + debian? ? "#{driver_pkg_name}_1.0-1_#{arch}.deb" : "#{driver_pkg_name}-1.0-1.#{arch}.rpm" + end + cached(:cuda_pkg_file) do + debian? ? "#{cuda_pkg_name}_#{cuda_version}-#{cuda_suffix}-1_#{arch}.deb" : "#{cuda_pkg_name}-#{cuda_version}_#{cuda_suffix}-1.#{arch}.rpm" + end + + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) do |node| + node.override['cluster']['nvidia']['driver_base_url'] = driver_base_url + node.override['cluster']['nvidia']['cuda']['base_url'] = cuda_base_url + node.override['cluster']['nvidia']['cuda']['driver_version_suffix'] = cuda_suffix + node.override['cluster']['sources_dir'] = sources_dir + end + ConvergeNvidiaRepo.add(runner, driver_version: driver_version, cuda_version: cuda_version) + end + + cached(:resource) do + chef_run.find_resource('nvidia_repo', 'add') + end + + it 'computes the local repo platform' do + expect(resource.local_repo_platform).to eq(local_repo_platform) + end + + it 'computes the driver repo package name, file, path and url' do + expect(resource.driver_repo_package_name).to eq(driver_pkg_name) + expect(resource.driver_repo_package_file).to eq(driver_pkg_file) + expect(resource.driver_repo_package_path).to eq("#{sources_dir}/#{driver_pkg_file}") + expect(resource.driver_repo_source_url).to eq("#{driver_base_url}/#{driver_pkg_file}") + end + + it 'computes the cuda repo package name, file, path and url' do + expect(resource.cuda_repo_package_name).to eq(cuda_pkg_name) + expect(resource.cuda_repo_package_file).to eq(cuda_pkg_file) + expect(resource.cuda_repo_package_path).to eq("#{sources_dir}/#{cuda_pkg_file}") + expect(resource.cuda_repo_source_url).to eq("#{cuda_base_url}/#{cuda_pkg_file}") + end end end +end + +describe 'nvidia_repo:arch_suffix' do + for_all_oses do |platform, version| + context "on #{platform}#{version}" do + cached(:debian?) { platform == 'ubuntu' } + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(false) + runner(platform: platform, version: version, step_into: ['nvidia_repo']) + end + cached(:resource) do + ConvergeNvidiaRepo.add(chef_run) + chef_run.find_resource('nvidia_repo', 'add') + end + + it 'is the rpm/deb arch on x86' do + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + expect(resource.arch_suffix).to eq(debian? ? 'amd64' : 'x86_64') + end - context 'when in region other than China' do - cached(:resource) do - ConvergeNvidiaRepo.add(chef_run, aws_region: 'anything') - chef_run.find_resource('nvidia_repo', 'add') + it 'is the rpm/deb arch on arm' do + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) + expect(resource.arch_suffix).to eq(debian? ? 'arm64' : 'aarch64') + end end + end +end + +describe 'nvidia_repo:add' do + for_all_oses do |platform, version| + context "on #{platform}#{version}" do + cached(:driver_version) { '999.88.77' } + cached(:cuda_version) { '13.0.2' } + cached(:cuda_suffix) { '580.95.05' } + cached(:driver_base_url) { 'https://driver.example/nvidia_driver' } + cached(:cuda_base_url) { 'https://cuda.example/cuda' } + cached(:sources_dir) { '/fake/sources' } + + cached(:debian?) { platform == 'ubuntu' } + cached(:local_repo_platform) { nvidia_local_repo_platform_for(platform, version) } + cached(:arch) { debian? ? 'amd64' : 'x86_64' } + cached(:driver_pkg_name) { "nvidia-driver-local-repo-#{local_repo_platform}-#{driver_version}" } + cached(:cuda_pkg_name) { "cuda-repo-#{local_repo_platform}-13-0-local" } + cached(:driver_pkg_file) do + debian? ? "#{driver_pkg_name}_1.0-1_#{arch}.deb" : "#{driver_pkg_name}-1.0-1.#{arch}.rpm" + end + cached(:cuda_pkg_file) do + debian? ? "#{cuda_pkg_name}_#{cuda_version}-#{cuda_suffix}-1_#{arch}.deb" : "#{cuda_pkg_name}-#{cuda_version}_#{cuda_suffix}-1.#{arch}.rpm" + end + cached(:driver_pkg_path) { "#{sources_dir}/#{driver_pkg_file}" } + cached(:cuda_pkg_path) { "#{sources_dir}/#{cuda_pkg_file}" } + + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) + allow_any_instance_of(Object).to receive(:on_docker?).and_return(false) + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + mock_file_exists('/usr/bin/nvidia-smi', false) + mock_file_exists('/usr/local/cuda', false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) do |node| + node.override['cluster']['nvidia']['driver_base_url'] = driver_base_url + node.override['cluster']['nvidia']['cuda']['base_url'] = cuda_base_url + node.override['cluster']['nvidia']['cuda']['driver_version_suffix'] = cuda_suffix + node.override['cluster']['sources_dir'] = sources_dir + end + ConvergeNvidiaRepo.add(runner, driver_version: driver_version, cuda_version: cuda_version) + end + + it 'downloads the driver and cuda local-repo installers' do + is_expected.to create_remote_file(driver_pkg_path).with( + source: "#{driver_base_url}/#{driver_pkg_file}", mode: '0644', retries: 3, retry_delay: 5 + ) + is_expected.to create_remote_file(cuda_pkg_path).with( + source: "#{cuda_base_url}/#{cuda_pkg_file}", mode: '0644', retries: 3, retry_delay: 5 + ) + end + + if platform == 'ubuntu' + it 'installs the local-repo debs, enrolls keys and refreshes apt once' do + is_expected.to install_dpkg_package(driver_pkg_name).with(source: driver_pkg_path) + is_expected.to install_dpkg_package(cuda_pkg_name).with(source: cuda_pkg_path) + is_expected.to run_execute("Install keyring for #{driver_pkg_name}") + is_expected.to run_execute("Install keyring for #{cuda_pkg_name}") + is_expected.to update_apt_update('Update apt cache for NVIDIA local repos') + end + + it 'refreshes the apt cache only once for all local repos' do + apt_updates = chef_run.run_context.resource_collection.select do |r| + r.resource_name == :apt_update + end + expect(apt_updates.size).to eq(1) + end + else + it 'installs the local-repo rpms and refreshes the dnf cache once' do + is_expected.to install_rpm_package(driver_pkg_name).with(source: driver_pkg_path) + is_expected.to install_rpm_package(cuda_pkg_name).with(source: cuda_pkg_path) + is_expected.to run_execute('Refresh dnf metadata for NVIDIA local repos').with(command: 'dnf clean all') + is_expected.to flush_cache_dnf_package('Update dnf cache for NVIDIA local repos') + end - it 'is cn' do - expect(resource.repo_domain).to eq('com') + it 'runs dnf clean all only once for all local repos' do + clean_alls = chef_run.run_context.resource_collection.select do |r| + r.resource_name == :execute && r.name == 'Refresh dnf metadata for NVIDIA local repos' + end + expect(clean_alls.size).to eq(1) + end + end end end end -describe 'nvidia_repo:arch_suffix' do - cached(:chef_run) do - ChefSpec::SoloRunner.new(step_into: ['nvidia_repo']) +describe 'nvidia_repo:add skip conditions' do + def converge_add(nvidia_enabled:, on_docker:, nvidia_smi:, cuda:) + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(nvidia_enabled) + allow_any_instance_of(Object).to receive(:on_docker?).and_return(on_docker) + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + mock_file_exists('/usr/bin/nvidia-smi', nvidia_smi) + mock_file_exists('/usr/local/cuda', cuda) + ConvergeNvidiaRepo.add(runner(platform: 'amazon', version: '2023', step_into: ['nvidia_repo'])) end - context 'when on arm' do - cached(:resource) do - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(true) - ConvergeNvidiaRepo.add(chef_run) - chef_run.find_resource('nvidia_repo', 'add') + context 'when nvidia is not enabled' do + cached(:converged) { converge_add(nvidia_enabled: false, on_docker: false, nvidia_smi: false, cuda: false) } + + it 'does not add any repo' do + expect(converged).not_to install_rpm_package(/nvidia-driver-local-repo/) + expect(converged).not_to install_rpm_package(/cuda-repo/) end + end + + context 'when on docker' do + cached(:converged) { converge_add(nvidia_enabled: true, on_docker: true, nvidia_smi: false, cuda: false) } - it 'is sbsa' do - expect(resource.arch_suffix).to eq('sbsa') + it 'does not add any repo' do + expect(converged).not_to install_rpm_package(/nvidia-driver-local-repo/) + expect(converged).not_to install_rpm_package(/cuda-repo/) end end - context 'when on x86' do - cached(:resource) do - allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) - ConvergeNvidiaRepo.add(chef_run) - chef_run.find_resource('nvidia_repo', 'add') + context 'when the driver is already installed but cuda is not' do + cached(:converged) { converge_add(nvidia_enabled: true, on_docker: false, nvidia_smi: true, cuda: false) } + + it 'skips the driver repo but still adds the cuda repo' do + expect(converged).not_to install_rpm_package(/nvidia-driver-local-repo/) + expect(converged).to install_rpm_package(/cuda-repo-amzn2023-13-0-local/) end + end + + context 'when cuda is already installed but the driver is not' do + cached(:converged) { converge_add(nvidia_enabled: true, on_docker: false, nvidia_smi: false, cuda: true) } - it 'is x86_64' do - expect(resource.arch_suffix).to eq('x86_64') + it 'skips the cuda repo but still adds the driver repo' do + expect(converged).to install_rpm_package(/nvidia-driver-local-repo/) + expect(converged).not_to install_rpm_package(/cuda-repo-amzn2023-13-0-local/) end end end -describe 'nvidia_repo:add' do - for_all_oses do |platform, version| - context "on #{platform}#{version}" do - cached(:repo_domain) { 'repo_domain' } - cached(:arch_suffix) { 'arch_suffix' } - cached(:nvidia_platform) do - case platform - when 'amazon' - version == '2023' ? 'rhel9' : 'rhel7' - when 'centos' - 'rhel7' - when 'redhat', 'rocky' - "rhel#{version.to_i}" - when 'ubuntu' - "ubuntu#{version.delete('.')}" - end - end - cached(:repository_url) { "https://developer.download.nvidia.#{repo_domain}/compute/cuda/repos/#{nvidia_platform}/#{arch_suffix}" } - cached(:repository_key) { platform == 'ubuntu' ? '3bf863cc.pub' : 'D42D0685.pub' } - cached(:chef_run) do - runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) - stubs_for_resource('nvidia_repo') do |res| - allow(res).to receive(:repo_domain).and_return(repo_domain) - allow(res).to receive(:arch_suffix).and_return(arch_suffix) - end - ConvergeNvidiaRepo.add(runner) - end - - it 'adds nvidia repo' do - is_expected.to add_nvidia_repo('add') - is_expected.to update_package_repos('update package repos') - is_expected.to add_package_repos('add nvidia-repo').with( - repo_name: "nvidia-repo", - baseurl: repository_url, - gpgkey: "#{repository_url}/#{repository_key}", - disable_modularity: true - ) - end +describe 'nvidia_repo: overriding base_url to the official NVIDIA URLs' do + cached(:driver_version) { '580.105.08' } + cached(:cuda_version) { '13.0.2' } + cached(:cuda_suffix) { '580.95.05' } + cached(:driver_base_url) { "https://developer.download.nvidia.com/compute/nvidia-driver/#{driver_version}/local_installers" } + cached(:cuda_base_url) { "https://developer.download.nvidia.com/compute/cuda/#{cuda_version}/local_installers" } + cached(:driver_pkg_file) { "nvidia-driver-local-repo-amzn2023-#{driver_version}-1.0-1.x86_64.rpm" } + cached(:cuda_pkg_file) { "cuda-repo-amzn2023-13-0-local-#{cuda_version}_#{cuda_suffix}-1.x86_64.rpm" } + + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:nvidia_enabled?).and_return(true) + allow_any_instance_of(Object).to receive(:on_docker?).and_return(false) + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + mock_file_exists('/usr/bin/nvidia-smi', false) + mock_file_exists('/usr/local/cuda', false) + runner = runner(platform: 'amazon', version: '2023', step_into: ['nvidia_repo']) do |node| + node.override['cluster']['nvidia']['driver_base_url'] = driver_base_url + node.override['cluster']['nvidia']['cuda']['base_url'] = cuda_base_url + node.override['cluster']['nvidia']['cuda']['driver_version_suffix'] = cuda_suffix end + ConvergeNvidiaRepo.add(runner, driver_version: driver_version, cuda_version: cuda_version) + end + + it 'downloads the driver local repo from the official NVIDIA URL' do + is_expected.to create_remote_file(%r{/#{driver_pkg_file}$}).with( + source: "#{driver_base_url}/#{driver_pkg_file}" + ) + end + + it 'downloads the cuda local repo from the official NVIDIA URL' do + is_expected.to create_remote_file(%r{/#{cuda_pkg_file}$}).with( + source: "#{cuda_base_url}/#{cuda_pkg_file}" + ) end end describe 'nvidia_repo:remove' do for_all_oses do |platform, version| context "on #{platform}#{version}" do - cached(:repo_domain) { 'repo_domain' } - cached(:arch_suffix) { 'arch_suffix' } - cached(:nvidia_platform) do - case platform - when 'amazon', 'centos' - 'rhel7' - when 'redhat' - "rhel#{version}" - when 'ubuntu' - "ubuntu#{version.delete('.')}" - end - end - cached(:repository_url) { "https://developer.download.nvidia.#{repo_domain}/compute/cuda/repos/#{nvidia_platform}/#{arch_suffix}" } - cached(:repository_key) { platform == 'ubuntu' ? '3bf863cc.pub' : 'D42D0685.pub' } - cached(:chef_run) do - runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) - stubs_for_resource('nvidia_repo') do |res| - allow(res).to receive(:repo_domain).and_return(repo_domain) - allow(res).to receive(:arch_suffix).and_return(arch_suffix) + cached(:driver_version) { '999.88.77' } + cached(:cuda_version) { '13.0.2' } + cached(:cuda_suffix) { '580.95.05' } + cached(:sources_dir) { '/fake/sources' } + cached(:debian?) { platform == 'ubuntu' } + cached(:local_repo_platform) { nvidia_local_repo_platform_for(platform, version) } + cached(:arch) { debian? ? 'amd64' : 'x86_64' } + cached(:driver_pkg_name) { "nvidia-driver-local-repo-#{local_repo_platform}-#{driver_version}" } + cached(:cuda_pkg_name) { "cuda-repo-#{local_repo_platform}-13-0-local" } + cached(:driver_pkg_file) do + debian? ? "#{driver_pkg_name}_1.0-1_#{arch}.deb" : "#{driver_pkg_name}-1.0-1.#{arch}.rpm" + end + cached(:cuda_pkg_file) do + debian? ? "#{cuda_pkg_name}_#{cuda_version}-#{cuda_suffix}-1_#{arch}.deb" : "#{cuda_pkg_name}-#{cuda_version}_#{cuda_suffix}-1.#{arch}.rpm" + end + + context 'when this run added the repos' do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) do |node| + node.override['cluster']['nvidia']['cuda']['driver_version_suffix'] = cuda_suffix + node.override['cluster']['sources_dir'] = sources_dir + end + runner.node.run_state['nvidia_driver_repo_added'] = true + runner.node.run_state['nvidia_cuda_repo_added'] = true + chef_run_remove(runner) + end + + def chef_run_remove(runner) + runner.converge_dsl('aws-parallelcluster-platform') do + nvidia_repo 'remove' do + driver_version '999.88.77' + cuda_version '13.0.2' + action :remove + end + end + end + + it 'removes the local-repo packages and deletes the installers' do + if debian? + is_expected.to purge_package(driver_pkg_name) + is_expected.to purge_package(cuda_pkg_name) + else + is_expected.to remove_package(driver_pkg_name) + is_expected.to remove_package(cuda_pkg_name) + end + is_expected.to delete_file("#{sources_dir}/#{driver_pkg_file}") + is_expected.to delete_file("#{sources_dir}/#{cuda_pkg_file}") + end + + it 'refreshes the package manager metadata after removal' do + if debian? + is_expected.to update_apt_update('Refresh apt metadata after removing NVIDIA local repos') + else + is_expected.to run_execute('Refresh dnf metadata after removing NVIDIA local repos').with( + command: 'dnf clean all' + ) + end + end + + it 'refreshes the package manager metadata after each repo removal' do + ordered_names = chef_run.run_context.resource_collection.map(&:name) + refresh_name = if debian? + 'Refresh apt metadata after removing NVIDIA local repos' + else + 'Refresh dnf metadata after removing NVIDIA local repos' + end + + driver_remove_index = ordered_names.index(driver_pkg_name) + cuda_remove_index = ordered_names.index(cuda_pkg_name) + refresh_indexes = ordered_names.each_index.select { |i| ordered_names[i] == refresh_name } + + expect(driver_remove_index).not_to be_nil + expect(cuda_remove_index).not_to be_nil + # One refresh per removed repo, and each refresh comes after a removal. + expect(refresh_indexes.size).to eq(2) + expect(refresh_indexes.any? { |i| i > driver_remove_index }).to be(true) + expect(refresh_indexes.any? { |i| i > cuda_remove_index }).to be(true) end - ConvergeNvidiaRepo.remove(runner) end - it 'removes nvidia repo' do - is_expected.to remove_nvidia_repo('remove') - is_expected.to remove_package_repos('remove nvidia-repo').with_repo_name('nvidia-repo') + context 'when this run added only the driver repo' do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) do |node| + node.override['cluster']['nvidia']['cuda']['driver_version_suffix'] = cuda_suffix + node.override['cluster']['sources_dir'] = sources_dir + end + runner.node.run_state['nvidia_driver_repo_added'] = true + runner.converge_dsl('aws-parallelcluster-platform') do + nvidia_repo 'remove' do + driver_version '999.88.77' + cuda_version '13.0.2' + action :remove + end + end + end + + it 'removes only the driver repo, leaving the cuda repo untouched' do + if debian? + is_expected.to purge_package(driver_pkg_name) + is_expected.not_to purge_package(cuda_pkg_name) + else + is_expected.to remove_package(driver_pkg_name) + is_expected.not_to remove_package(cuda_pkg_name) + end + end + + it 'still refreshes the package manager metadata after removing the driver repo' do + if debian? + is_expected.to update_apt_update('Refresh apt metadata after removing NVIDIA local repos') + else + is_expected.to run_execute('Refresh dnf metadata after removing NVIDIA local repos').with( + command: 'dnf clean all' + ) + end + end + end + + context 'when this run did not add the repos' do + cached(:chef_run) do + allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false) + runner = runner(platform: platform, version: version, step_into: ['nvidia_repo']) + runner.converge_dsl('aws-parallelcluster-platform') do + nvidia_repo 'remove' do + action :remove + end + end + end + + it 'removes nothing' do + is_expected.not_to remove_package(driver_pkg_name) + is_expected.not_to remove_package(cuda_pkg_name) + is_expected.not_to purge_package(driver_pkg_name) + is_expected.not_to purge_package(cuda_pkg_name) + end + + it 'does not refresh the package manager metadata' do + is_expected.not_to update_apt_update('Refresh apt metadata after removing NVIDIA local repos') + is_expected.not_to run_execute('Refresh dnf metadata after removing NVIDIA local repos') + end end end end diff --git a/cookbooks/aws-parallelcluster-platform/templates/nvidia/cuda.sh.erb b/cookbooks/aws-parallelcluster-platform/templates/nvidia/cuda.sh.erb new file mode 100644 index 0000000000..6d05136910 --- /dev/null +++ b/cookbooks/aws-parallelcluster-platform/templates/nvidia/cuda.sh.erb @@ -0,0 +1,3 @@ +# Added by AWS ParallelCluster: expose the CUDA toolkit to all users +export PATH=<%= @cuda_path %>/bin:$PATH +export LD_LIBRARY_PATH=<%= @cuda_path %>/lib64:$LD_LIBRARY_PATH diff --git a/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_fabric_manager_spec.rb b/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_fabric_manager_spec.rb index 242ce90e5f..9a40b62d83 100644 --- a/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_fabric_manager_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_fabric_manager_spec.rb @@ -12,15 +12,14 @@ control 'tag:install_expected_versions_of_nvidia_fabric_manager_installed' do only_if { !os_properties.arm? && ['yes', true, 'true'].include?(node['cluster']['nvidia']['enabled']) } + # On images that ship their own NVIDIA stack (e.g. DLAMI) pcluster skips the + # install and leaves Fabric Manager as-is, so the version attribute is unset and + # we only verify it is present. When pcluster installs it, the version attribute + # is set and we verify the package matches the configured version. describe package(node['cluster']['nvidia']['fabricmanager']['package']) do it { should be_installed } its('version') { should match /#{node['cluster']['nvidia']['fabricmanager']['version']}/ } end - - version_lock_check = os_properties.debian_family? ? 'apt-mark showhold | grep "nvidia-fabric.*manager"' : 'yum versionlock list | grep "nvidia-fabric.*manager"' - describe bash(version_lock_check) do - its('exit_status') { should eq 0 } - end end control 'tag:config_nvidia_fabric_manager_enabled' do diff --git a/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_spec.rb b/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_spec.rb index c2d165ce70..5106394dd6 100644 --- a/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/test/controls/nvidia_spec.rb @@ -15,9 +15,12 @@ (node['cluster']['nvidia']['enabled'] == 'yes' || node['cluster']['nvidia']['enabled'] == true) end + # driver_version holds either the version pcluster installed or, on images that + # ship their own driver (e.g. DLAMI), the detected pre-installed version. Either + # way the driver must report exactly this version, which verifies pcluster + # installed the configured version and did not change a shipped one. expected_nvidia_driver_version = node['cluster']['nvidia']['driver_version'] expected_nvidia_kernel_license = 'Dual MIT/GPL' - expected_nvidia_kernel_module = "NVRM version: NVIDIA UNIX Open Kernel Module" describe "nvidia driver version is expected to be #{expected_nvidia_driver_version}" do subject { command('modinfo -F version nvidia').stdout.strip } @@ -28,11 +31,6 @@ subject { command('modinfo -F license nvidia').stdout.strip } it { should eq expected_nvidia_kernel_license } end - - describe "nvidia kernel module is expected to be #{expected_nvidia_kernel_module}" do - subject { command('cat /proc/driver/nvidia/version').stdout.strip } - it { should include expected_nvidia_kernel_module } - end end control 'tag:install_expected_versions_of_nvidia_cuda_installed' do