[DCV] Retry dcv-gl offline rpm installs to absorb rpm lock contention

The 'install dcv-gl dependencies offline' and 'install dcv-gl offline' execs
sporadically failed with 'can't create transaction lock on /var/lib/rpm/.rpm.lock
(Resource temporarily unavailable)' when a background dnf actor (dnf-makecache
timer / PackageKit) held the rpm DB lock. Add
retries 3 / retry_delay 5 (matching the sibling 'download dcv-gl dependencies'
resource) so the install re-runs once the lock frees. Spec asserts the retry
attributes and stubs the only_if dir guard.

Author: Himani Anil Deshpande

cookbooks/aws-parallelcluster-platform/resources/dcv/partial/_rhel_common.rb

@@ -136,12 +136,16 @@ def install_dcv_gl
     execute 'install dcv-gl dependencies offline' do
       command "rpm -ivh #{dcv_gl_deps_dir}/*.rpm"
       only_if { ::Dir.exist?(dcv_gl_deps_dir) && !::Dir.empty?(dcv_gl_deps_dir) }
+      retries 3
+      retry_delay 5
     end
 
     package = "#{node['cluster']['sources_dir']}/#{dcv_package}/#{dcv_gl}"
     # Install dcv-gl without repo access
     execute 'install dcv-gl offline' do
       command "rpm -ivh #{package}"
+      retries 3
+      retry_delay 5
     end
   end
 end

cookbooks/aws-parallelcluster-platform/spec/unit/resources/dcv_spec.rb

@@ -849,6 +849,11 @@ def self.nothing(chef_run)
           allow_any_instance_of(Object).to receive(:arm_instance?).and_return(false)
           allow_any_instance_of(Object).to receive(:graphic_instance?).and_return(true)
           allow_any_instance_of(Object).to receive(:nvidia_installed?).and_return(true)
+          dcv_gl_deps_dir = "#{sources_dir}/dcv-gl-deps"
+          allow(::Dir).to receive(:exist?).and_call_original
+          allow(::Dir).to receive(:empty?).and_call_original
+          allow(::Dir).to receive(:exist?).with(dcv_gl_deps_dir).and_return(true)
+          allow(::Dir).to receive(:empty?).with(dcv_gl_deps_dir).and_return(false)
           ConvergeDcv.configure(runner)
         end
         cached(:node) { chef_run.node }
@@ -868,8 +873,14 @@ def self.nothing(chef_run)
             is_expected.to run_execute('apt install dcv-gl')
               .with_command("apt -y install #{sources_dir}/#{dcv_package}/#{dcv_gl}")
           else
+            is_expected.to run_execute('install dcv-gl dependencies offline')
+              .with_command("rpm -ivh #{sources_dir}/dcv-gl-deps/*.rpm")
+              .with_retries(3)
+              .with_retry_delay(5)
             is_expected.to run_execute('install dcv-gl offline')
               .with_command("rpm -ivh #{sources_dir}/#{dcv_package}/#{dcv_gl}")
+              .with_retries(3)
+              .with_retry_delay(5)
           end
         end