Retry transient upstream errors in snippet and LOC generation (#8425)

Rescue 502/503/504 errors from the snippet generator and lines-of-code
counter services, requeuing the job up to 3 times with a random delay.
After retries are exhausted, the exception propagates to Sidekiq/Sentry
so persistent failures are still surfaced.

Closes #8396

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: iHiD

app/commands/iteration/count_lines_of_code.rb

@@ -3,7 +3,9 @@ class Iteration::CountLinesOfCode
 
   queue_as :solution_processing
 
-  initialize_with :iteration
+  initialize_with :iteration, retries_count: 0
+
+  MAX_RETRIES = 3
 
   def call
     # Sometimes the iteration might be deleted
@@ -34,6 +36,10 @@ def call
 
     iteration.update_column(:num_loc, num_loc)
     Solution::UpdateNumLoc.(iteration.solution)
+  rescue RestClient::BadGateway, RestClient::ServiceUnavailable, RestClient::GatewayTimeout
+    raise if retries_count >= MAX_RETRIES
+
+    self.class.defer(iteration, retries_count: retries_count + 1, wait: rand(30..90))
   ensure
     ToolingJob::DeleteFromEFS.(efs_dir)
   end

app/commands/iteration/generate_snippet.rb

@@ -3,7 +3,9 @@ class Iteration::GenerateSnippet
 
   queue_as :solution_processing
 
-  initialize_with :iteration
+  initialize_with :iteration, retries_count: 0
+
+  MAX_RETRIES = 3
 
   def call
     # Sometimes the iteration might be deleted
@@ -35,5 +37,9 @@ def call
     return if e.message.include?("Invalid Unicode")
 
     raise
+  rescue RestClient::BadGateway, RestClient::ServiceUnavailable, RestClient::GatewayTimeout
+    raise if retries_count >= MAX_RETRIES
+
+    self.class.defer(iteration, retries_count: retries_count + 1, wait: rand(30..90))
   end
 end

test/commands/iteration/count_lines_of_code_test.rb

@@ -98,6 +98,31 @@ class Iteration::CountLinesOfCodeTest < ActiveJob::TestCase
     assert_equal iteration.num_loc, solution.reload.num_loc
   end
 
+  test "requeues on bad gateway" do
+    submission = create :submission
+    create :submission_file, submission:, content: "Some source code"
+    iteration = create(:iteration, submission:)
+
+    RestClient.stubs(:post).raises(RestClient::BadGateway)
+
+    Iteration::CountLinesOfCode.expects(:defer).with do |iter, **kwargs|
+      iter == iteration && kwargs[:retries_count] == 1 && kwargs[:wait].between?(30, 90)
+    end
+    Iteration::CountLinesOfCode.(iteration)
+  end
+
+  test "raises after max retries exhausted" do
+    submission = create :submission
+    create :submission_file, submission:, content: "Some source code"
+    iteration = create(:iteration, submission:)
+
+    RestClient.stubs(:post).raises(RestClient::BadGateway)
+
+    assert_raises RestClient::BadGateway do
+      Iteration::CountLinesOfCode.(iteration, retries_count: 3)
+    end
+  end
+
   test "solution is not updated if another iteration is published" do
     solution = create :concept_solution
     submission = create :submission

test/commands/iteration/generate_snippet_test.rb

@@ -77,6 +77,49 @@ class Iteration::GenerateSnippetTest < ActiveJob::TestCase
     Iteration::GenerateSnippet.(iteration)
   end
 
+  test "requeues on bad gateway" do
+    iteration = create :iteration, submission: @submission
+
+    RestClient.stubs(:post).raises(RestClient::BadGateway)
+
+    Iteration::GenerateSnippet.expects(:defer).with do |iter, **kwargs|
+      iter == iteration && kwargs[:retries_count] == 1 && kwargs[:wait].between?(30, 90)
+    end
+    Iteration::GenerateSnippet.(iteration)
+  end
+
+  test "requeues on service unavailable" do
+    iteration = create :iteration, submission: @submission
+
+    RestClient.stubs(:post).raises(RestClient::ServiceUnavailable)
+
+    Iteration::GenerateSnippet.expects(:defer).with do |iter, **kwargs|
+      iter == iteration && kwargs[:retries_count] == 1 && kwargs[:wait].between?(30, 90)
+    end
+    Iteration::GenerateSnippet.(iteration)
+  end
+
+  test "requeues on gateway timeout" do
+    iteration = create :iteration, submission: @submission
+
+    RestClient.stubs(:post).raises(RestClient::GatewayTimeout)
+
+    Iteration::GenerateSnippet.expects(:defer).with do |iter, **kwargs|
+      iter == iteration && kwargs[:retries_count] == 1 && kwargs[:wait].between?(30, 90)
+    end
+    Iteration::GenerateSnippet.(iteration)
+  end
+
+  test "raises after max retries exhausted" do
+    iteration = create :iteration, submission: @submission
+
+    RestClient.stubs(:post).raises(RestClient::BadGateway)
+
+    assert_raises RestClient::BadGateway do
+      Iteration::GenerateSnippet.(iteration, retries_count: 3)
+    end
+  end
+
   test "handle long snippets" do
     code = "Some source code"
     snippet = "Some generated snippet#{"x" * 1500}"