feat: make exponential backoff base and max intervals configurable

Author: raheelshahzad

crates/brightstaff/src/handlers/llm.rs

@@ -404,8 +404,11 @@ async fn llm_chat_inner(
                 let max_retries = provider.max_retries.unwrap_or(1);
                 if provider.retry_on_ratelimit == Some(true) && (attempts - 1) < max_retries as usize
                 {
-                    // Exponential backoff: base interval 25ms
-                    let backoff_ms = 25 * 2u64.pow(attempts as u32 - 1);
+                    // Configurable exponential backoff
+                    let base_ms = provider.retry_backoff_base_ms.unwrap_or(25);
+                    let max_ms = provider.retry_backoff_max_ms.unwrap_or(1000);
+                    let backoff_ms = (base_ms * 2u64.pow(attempts as u32 - 1)).min(max_ms);
+
                     debug!(
                         request_id = %request_id,
                         "429 received, retrying after {}ms (attempt {})",

crates/common/src/configuration.rs

@@ -331,6 +331,8 @@ pub struct LlmProvider {
     pub retry_on_ratelimit: Option<bool>,
     pub max_retries: Option<u32>,
     pub retry_to_same_provider: Option<bool>,
+    pub retry_backoff_base_ms: Option<u64>,
+    pub retry_backoff_max_ms: Option<u64>,
 }
 
 pub trait IntoModels {
@@ -378,6 +380,8 @@ impl Default for LlmProvider {
             retry_on_ratelimit: None,
             max_retries: None,
             retry_to_same_provider: None,
+            retry_backoff_base_ms: None,
+            retry_backoff_max_ms: None,
         }
     }
 }

crates/common/src/llm_providers.rs

@@ -313,6 +313,8 @@ mod tests {
             retry_on_ratelimit: None,
             max_retries: None,
             retry_to_same_provider: None,
+            retry_backoff_base_ms: None,
+            retry_backoff_max_ms: None,
         }
     }
 

docs/source/resources/includes/arch_config_full_reference.yaml

@@ -40,6 +40,8 @@ model_providers:
     retry_on_ratelimit: true
     max_retries: 2
     retry_to_same_provider: false # If false, Arch will pick another random model from the list
+    retry_backoff_base_ms: 25 # Base delay for exponential backoff
+    retry_backoff_max_ms: 1000 # Maximum delay for exponential backoff
 
   # Example: Passthrough authentication for LiteLLM or similar proxies
   # When passthrough_auth is true, client's Authorization header is forwarded