Retry transient AWS credential errors in all jobs

app/jobs/application_job.rb

@@ -3,6 +3,12 @@ class ApplicationJob < ActiveJob::Base
   # (excluded via Sentry config's excluded_exceptions)
   retry_on ActiveRecord::Deadlocked
 
+  # Retry jobs that fail due to transient AWS credential errors
+  # (e.g. during credential rotation or ECS task role refresh)
+  retry_on Aws::Sigv4::Errors::MissingCredentialsError,
+    Aws::Errors::MissingCredentialsError,
+    wait: 10.seconds, attempts: 3
+
   # Can be overriden to disable this
   def guard_against_deserialization_errors? = true
   rescue_from ActiveJob::DeserializationError do |exception|

test/jobs/application_job_test.rb

@@ -7,6 +7,18 @@ def perform
     end
   end
 
+  class TestAwsSigv4MissingCredentialsJob < ApplicationJob
+    def perform
+      raise Aws::Sigv4::Errors::MissingCredentialsError
+    end
+  end
+
+  class TestAwsMissingCredentialsJob < ApplicationJob
+    def perform
+      raise Aws::Errors::MissingCredentialsError, "credentials not set"
+    end
+  end
+
   class TestDeserializationJob < ApplicationJob
     def perform
       # ActiveJob::DeserializationError uses $! so this needs
@@ -30,6 +42,18 @@ def perform(user)
     end
   end
 
+  test "Aws::Sigv4 missing credentials retries the job" do
+    assert_enqueued_with(job: TestAwsSigv4MissingCredentialsJob) do
+      TestAwsSigv4MissingCredentialsJob.perform_now
+    end
+  end
+
+  test "Aws::Errors missing credentials retries the job" do
+    assert_enqueued_with(job: TestAwsMissingCredentialsJob) do
+      TestAwsMissingCredentialsJob.perform_now
+    end
+  end
+
   test "Deserialization raises for non-active-record exception" do
     assert_raises ActiveJob::DeserializationError do
       TestDeserializationJob.perform_now