[Build] Further Mitigate build image failures in RHEL/Rocky caused by out of sync repositories

hanwen-cluster · hanwen-cluster · commit 5b6afd98d4c1 · 2026-05-21T09:49:06.000-07:00
Generalizes the approach created by #3060 Introduces a `robust_package` resource that wraps the Chef `package` resource with a metadata-refresh retry strategy on RHEL/Rocky to mitigate transient build failures from out-of-sync RHUI mirrors. On Amazon Linux and Debian-based platforms it falls back to the stock `package` resource with retries, so behavior is unchanged. In addition, this commit does some refactor to reduce code duplication.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **ENHANCEMENTS**
 - Improve resilience of EBS volume attachment during cluster creation by retrying on transient IMDS connectivity failures.
+- Further reduce transient build-image failures on RHEL and Rocky caused by out-of-sync repo mirrors by resetting metadata upon retry.
 
 **BUG FIXES**
 - Fix cluster creation failure caused by Slurm accounting bootstrap failing when ClusterName is overridden 
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efa/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efa/partial/_common.rb
@@ -45,9 +45,8 @@
   package_repos 'update package repos' do
     action :update
   end
-  package prerequisites do
-    retries 3
-    retry_delay 5
+  robust_package 'install efa prerequisites' do
+    packages prerequisites
   end
 
   action_download_and_install
diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb
@@ -23,9 +23,8 @@
     action :update
   end
 
-  package prerequisites do
-    retries 3
-    retry_delay 5
+  robust_package 'install efs-utils prerequisites' do
+    packages prerequisites
   end
 
   directory node['cluster']['sources_dir'] do
diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efa_spec.rb
@@ -56,7 +56,7 @@ def self.configure(chef_run)
         if %(redhat rocky).include?(platform) || platform == 'amazon' && version == '2023'
           %w(environment-modules libibverbs-utils librdmacm-utils rdma-core-devel)
         else
-          "environment-modules"
+          %w(environment-modules)
         end
       end
       let(:chef_run) do
@@ -98,7 +98,8 @@ def self.configure(chef_run)
             is_expected.not_to write_log('efa installed')
             is_expected.not_to remove_package(%w(openmpi-devel openmpi))
             is_expected.to update_package_repos('update package repos')
-            is_expected.to install_package(prerequisites)
+            is_expected.to install_robust_package('install efa prerequisites')
+              .with(packages: prerequisites)
             is_expected.to create_if_missing_remote_file("#{source_dir}/aws-efa-installer.tar.gz")
             is_expected.not_to run_bash('install efa')
           end
@@ -124,7 +125,8 @@ def self.configure(chef_run)
             is_expected.not_to write_log('efa installed')
             is_expected.to remove_package(platform == 'ubuntu' ? ['libopenmpi-dev'] : %w(openmpi-devel openmpi))
             is_expected.to update_package_repos('update package repos')
-            is_expected.to install_package(prerequisites)
+            is_expected.to install_robust_package('install efa prerequisites')
+              .with(packages: prerequisites)
             is_expected.to create_if_missing_remote_file("#{source_dir}/aws-efa-installer.tar.gz")
               .with(source: "https://efa-installer.amazonaws.com/aws-efa-installer-#{efa_version}.tar.gz")
               .with(mode: '0644')
diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb
@@ -155,9 +155,8 @@ def mock_already_installed(package, expected_version, installed)
         end
 
         it 'installs prerequisites' do
-          is_expected.to install_package(required_packages[platform])
-            .with(retries: 3)
-            .with(retry_delay: 5)
+          is_expected.to install_robust_package('install efs-utils prerequisites')
+            .with(packages: required_packages[platform])
         end
 
         it 'downloads tarball' do
diff --git a/cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_rhel.rb b/cookbooks/aws-parallelcluster-platform/resources/enroot/partial/_enroot_rhel.rb
@@ -15,9 +15,8 @@
 action :install_package do
   return unless nvidia_enabled? || nvidia_installed?
 
-  package prerequisites do
-    retries 3
-    retry_delay 5
+  robust_package 'install enroot prerequisites' do
+    packages prerequisites
   end
 
   bash "Install enroot" do
diff --git a/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/partial/_gdrcopy_common.rb b/cookbooks/aws-parallelcluster-platform/resources/gdrcopy/partial/_gdrcopy_common.rb
@@ -51,9 +51,8 @@ def gdrcopy_checksum
     action :update
   end
 
-  package gdrcopy_build_dependencies do
-    retries 3
-    retry_delay 5
+  robust_package 'install gdrcopy build dependencies' do
+    packages gdrcopy_build_dependencies
   end
 
   bash 'Install NVIDIA GDRCopy' do
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_alinux2023.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_alinux2023.rb
@@ -17,7 +17,6 @@
 end
 
 use 'partial/_install_packages_common.rb'
-use 'partial/_install_packages_rhel_amazon.rb'
 
 action :install do
   package new_resource.packages do
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_redhat8.rb
@@ -17,7 +17,6 @@
 end
 
 use 'partial/_install_packages_common.rb'
-use 'partial/_install_packages_rhel_rocky.rb'
 
 def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_rocky8.rb
@@ -17,7 +17,6 @@
 end
 
 use 'partial/_install_packages_common.rb'
-use 'partial/_install_packages_rhel_rocky.rb'
 
 def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_ubuntu22+.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/install_packages_ubuntu22+.rb
@@ -19,7 +19,6 @@
 default_action :setup
 
 use 'partial/_install_packages_common.rb'
-use 'partial/_install_packages_debian.rb'
 
 def default_packages
   # environment-modules required by EFA, Intel MPI and ARM PL
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_common.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_common.rb
@@ -19,6 +19,12 @@
          default: lazy { default_packages },
          description: 'Packages for the node'
 
+action :install do
+  robust_package "install_packages #{new_resource.name}" do
+    packages new_resource.packages
+  end
+end
+
 action :install_base_packages do
   install_packages 'default' do
     action :install
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_debian.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_debian.rb
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_rhel_amazon.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_rhel_amazon.rb
diff --git a/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_rhel_rocky.rb b/cookbooks/aws-parallelcluster-platform/resources/install_packages/partial/_install_packages_rhel_rocky.rb
diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/gdrcopy_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/gdrcopy_spec.rb
@@ -244,7 +244,8 @@ def self.configure(chef_run)
       end
 
       it 'builds dependencies' do
-        is_expected.to install_package(gdrcopy_dependencies).with_retries(3).with_retry_delay(5)
+        is_expected.to install_robust_package('install gdrcopy build dependencies')
+          .with(packages: gdrcopy_dependencies)
       end
 
       cached(:installation_code) { chef_run.bash('Install NVIDIA GDRCopy').code }
diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/install_packages_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/install_packages_spec.rb
@@ -40,16 +40,16 @@ def self.setup(chef_run)
               .with(flush_cache: { before: true })
           end
         else
-          it 'installs default packages with mirror refresh retry logic' do
-            is_expected.to run_ruby_block('install_packages_with_metadata_refresh')
+          it 'installs default packages via robust_package' do
+            is_expected.to install_robust_package('install_packages default')
+              .with(packages: default_packages)
           end
         end
 
       elsif platform == 'ubuntu'
-        it 'installs base packages' do
-          is_expected.to install_package(default_packages)
-            .with(retries: 10)
-            .with(retry_delay: 5)
+        it 'installs base packages via robust_package' do
+          is_expected.to install_robust_package('install_packages default')
+            .with(packages: default_packages)
         end
 
       else
diff --git a/cookbooks/aws-parallelcluster-shared/libraries/helpers.rb b/cookbooks/aws-parallelcluster-shared/libraries/helpers.rb
@@ -194,3 +194,42 @@ def run_cmd(cmd, timeout: nil, raise_on_error: false)
   Chef::Log.info("Stderr: #{result.stderr}") unless result.stderr.strip.empty?
   result
 end
+
+# Installs RPM packages on RHEL/Rocky with a metadata-refresh + mirror-rotation
+# retry strategy that mitigates transient failures caused by out-of-sync RHUI
+# mirrors (e.g. "No match for argument: <pkg>-<version>", or rpm transaction
+# lock contention with the dnf-makecache timer).
+#
+# Strategy:
+#   1. `dnf install -y --refresh <pkgs>` forces DNF to refresh metadata from
+#      the mirror it is currently using.
+#   2. `dnf clean metadata` between attempts forces DNF to re-evaluate
+#      available mirrors and potentially pick a different one next attempt.
+#
+# @param packages [Array<String>, String] Package names (optionally version-pinned)
+# @param max_retries [Integer] Maximum number of retry attempts (default: 10)
+# @param retry_delay [Integer] Seconds to wait between retries (default: 5)
+#
+# @example
+#   dnf_install_with_refresh(%w(dkms rpm-build check check-devel subunit))
+#
+def dnf_install_with_refresh(packages, max_retries: 10, retry_delay: 5)
+  packages = Array(packages)
+  Chef::Log.info("Installing packages with mirror refresh retry: #{packages.join(', ')}")
+
+  on_retry = lambda do |_attempt, _exception|
+    # This cleanup forces DNF to re-evaluate available mirrors
+    # and which mirror to use on the next attempt.
+    run_cmd('dnf clean metadata')
+  end
+
+  with_retries(
+    max_retries: max_retries,
+    retry_delay: retry_delay,
+    on_retry: on_retry
+  ) do
+    # --refresh forces DNF to refresh the metadata from the mirror it's currently using.
+    run_cmd("dnf install -y --refresh #{packages.join(' ')}", raise_on_error: true)
+    Chef::Log.info("Package installation succeeded")
+  end
+end
diff --git a/cookbooks/aws-parallelcluster-shared/resources/robust_package.rb b/cookbooks/aws-parallelcluster-shared/resources/robust_package.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+#
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+#
+# Resource:: robust_package
+#
+# Cross-platform wrapper around the Chef `package` resource that uses a
+# metadata-refresh + mirror-rotation retry strategy on RHEL/Rocky to mitigate
+# transient build-image failures caused by out-of-sync RHUI mirrors.
+#
+# On Amazon Linux and Debian-based platforms the resource falls back to the
+# stock Chef `package` resource (with retries) so behavior is unchanged there.
+#
+# @example
+#   robust_package 'install enroot prerequisites' do
+#     packages prerequisites
+#   end
+#
+
+provides :robust_package
+unified_mode true
+
+property :packages, [String, Array], required: true, name_property: false
+property :max_retries, Integer, default: 10
+property :retry_delay, Integer, default: 5
+
+default_action :install
+
+action :install do
+  packages = Array(new_resource.packages)
+  max_retries = new_resource.max_retries
+  retry_delay = new_resource.retry_delay
+
+  if platform?('redhat', 'rocky')
+    ruby_block "robust_package install #{new_resource.name}" do
+      block do
+        dnf_install_with_refresh(packages, max_retries: max_retries, retry_delay: retry_delay)
+      end
+    end
+  else
+    package packages do
+      retries max_retries
+      retry_delay retry_delay
+    end
+  end
+end
diff --git a/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/slurm_dependencies_spec.rb b/cookbooks/aws-parallelcluster-slurm/spec/unit/resources/slurm_dependencies_spec.rb
