From 0dac5e58330344ef5142c937dc93ea0190b6e8b1 Mon Sep 17 00:00:00 2001
From: Jeremy Walker <jez.walker@gmail.com>
Date: Fri, 13 Feb 2026 12:44:06 +0000
Subject: [PATCH] Retry transient AWS credential errors in all jobs

Add retry_on for Aws::Sigv4::Errors::MissingCredentialsError and
Aws::Errors::MissingCredentialsError to ApplicationJob, so any job
hitting a transient credential issue (e.g. during credential rotation
or ECS task role refresh) retries up to 3 times with 10s wait.
Persistent failures still raise to Sentry after retries exhaust.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 app/jobs/application_job.rb       |  6 ++++++
 test/jobs/application_job_test.rb | 24 ++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
index 45b91dabbe..b95c928640 100644
--- a/app/jobs/application_job.rb
+++ b/app/jobs/application_job.rb
@@ -3,6 +3,12 @@ class ApplicationJob < ActiveJob::Base
   # (excluded via Sentry config's excluded_exceptions)
   retry_on ActiveRecord::Deadlocked
 
+  # Retry jobs that fail due to transient AWS credential errors
+  # (e.g. during credential rotation or ECS task role refresh)
+  retry_on Aws::Sigv4::Errors::MissingCredentialsError,
+    Aws::Errors::MissingCredentialsError,
+    wait: 10.seconds, attempts: 3
+
   # Can be overriden to disable this
   def guard_against_deserialization_errors? = true
   rescue_from ActiveJob::DeserializationError do |exception|
diff --git a/test/jobs/application_job_test.rb b/test/jobs/application_job_test.rb
index 87a3d2ad39..6732bf96f9 100644
--- a/test/jobs/application_job_test.rb
+++ b/test/jobs/application_job_test.rb
@@ -7,6 +7,18 @@ def perform
     end
   end
 
+  class TestAwsSigv4MissingCredentialsJob < ApplicationJob
+    def perform
+      raise Aws::Sigv4::Errors::MissingCredentialsError
+    end
+  end
+
+  class TestAwsMissingCredentialsJob < ApplicationJob
+    def perform
+      raise Aws::Errors::MissingCredentialsError, "credentials not set"
+    end
+  end
+
   class TestDeserializationJob < ApplicationJob
     def perform
       # ActiveJob::DeserializationError uses $! so this needs
@@ -30,6 +42,18 @@ def perform(user)
     end
   end
 
+  test "Aws::Sigv4 missing credentials retries the job" do
+    assert_enqueued_with(job: TestAwsSigv4MissingCredentialsJob) do
+      TestAwsSigv4MissingCredentialsJob.perform_now
+    end
+  end
+
+  test "Aws::Errors missing credentials retries the job" do
+    assert_enqueued_with(job: TestAwsMissingCredentialsJob) do
+      TestAwsMissingCredentialsJob.perform_now
+    end
+  end
+
   test "Deserialization raises for non-active-record exception" do
     assert_raises ActiveJob::DeserializationError do
       TestDeserializationJob.perform_now