Skip to content

Commit 2d63e1e

Browse files
committed
feat(agent): add retry with exponential backoff for transient API errors
Transient errors (rate limits, timeouts, network issues) now retry up to 3 times with exponential backoff (1s, 2s, 4s). Non-recoverable errors (billing, auth) still exit immediately. Adds RetryConfig struct, get_transient_error_message(), and 8 new tests. Closes: fluent_cli-cd2
1 parent d1a2fa6 commit 2d63e1e

1 file changed

Lines changed: 192 additions & 27 deletions

File tree

crates/fluent-cli/src/agentic.rs

Lines changed: 192 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,58 @@ pub fn get_api_error_guidance(error_msg: &str) -> &'static str {
108108
}
109109
}
110110

111+
/// Configuration for retry logic on transient errors
112+
pub struct RetryConfig {
113+
/// Maximum number of retry attempts
114+
pub max_retries: u32,
115+
/// Initial delay in milliseconds before first retry
116+
pub initial_delay_ms: u64,
117+
/// Maximum delay in milliseconds between retries
118+
pub max_delay_ms: u64,
119+
/// Multiplier for exponential backoff
120+
pub backoff_multiplier: f64,
121+
}
122+
123+
impl Default for RetryConfig {
124+
fn default() -> Self {
125+
Self {
126+
max_retries: 3,
127+
initial_delay_ms: 1000, // 1 second
128+
max_delay_ms: 30000, // 30 seconds
129+
backoff_multiplier: 2.0,
130+
}
131+
}
132+
}
133+
134+
impl RetryConfig {
135+
/// Calculate delay for a given retry attempt (0-indexed)
136+
pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
137+
let delay_ms = (self.initial_delay_ms as f64
138+
* self.backoff_multiplier.powi(attempt as i32)) as u64;
139+
let capped_delay_ms = delay_ms.min(self.max_delay_ms);
140+
std::time::Duration::from_millis(capped_delay_ms)
141+
}
142+
}
143+
144+
/// Get user-friendly message for transient errors
145+
pub fn get_transient_error_message(error_msg: &str) -> &'static str {
146+
let lower = error_msg.to_lowercase();
147+
148+
if lower.contains("rate limit") || lower.contains("too many requests") || lower.contains("429")
149+
{
150+
"⏳ Rate limit hit. Waiting before retry..."
151+
} else if lower.contains("timeout") {
152+
"⏱️ Request timed out. Retrying..."
153+
} else if lower.contains("connection refused")
154+
|| lower.contains("network error")
155+
|| lower.contains("connection reset")
156+
{
157+
"🌐 Network error. Retrying..."
158+
} else {
159+
"🔄 Transient error. Retrying..."
160+
}
161+
}
162+
111163
/// Status of a todo item
112164
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
113165
pub enum TodoStatus {
@@ -1619,36 +1671,79 @@ impl<'a> AutonomousExecutor<'a> {
16191671
));
16201672
}
16211673

1622-
let reasoning_response = match self.perform_reasoning(iteration, max_iterations).await {
1623-
Ok(response) => response,
1624-
Err(e) => {
1625-
let error_msg = e.to_string();
1626-
let error_kind = classify_api_error(&error_msg);
1627-
1628-
match error_kind {
1629-
ApiErrorKind::NonRecoverable => {
1630-
// Log and exit immediately for non-recoverable errors
1631-
let guidance = get_api_error_guidance(&error_msg);
1632-
error!(
1633-
"agent.api.non_recoverable error='{}' guidance='{}'",
1634-
error_msg, guidance
1635-
);
1636-
self.tui.add_log(format!("🛑 {}", guidance));
1637-
self.tui.add_log(format!(
1638-
"❌ Agent stopping immediately due to non-recoverable API error"
1639-
));
1640-
return Err(anyhow!(
1641-
"Non-recoverable API error: {}. {}",
1642-
error_msg,
1643-
guidance
1644-
));
1645-
}
1646-
ApiErrorKind::Transient | ApiErrorKind::Unknown => {
1647-
// For transient errors, propagate normally (may retry)
1648-
return Err(e);
1674+
// Perform reasoning with retry logic for transient errors
1675+
let retry_config = RetryConfig::default();
1676+
let mut last_error: Option<anyhow::Error> = None;
1677+
1678+
let reasoning_response = 'retry_loop: {
1679+
for attempt in 0..=retry_config.max_retries {
1680+
match self.perform_reasoning(iteration, max_iterations).await {
1681+
Ok(response) => break 'retry_loop response,
1682+
Err(e) => {
1683+
let error_msg = e.to_string();
1684+
let error_kind = classify_api_error(&error_msg);
1685+
1686+
match error_kind {
1687+
ApiErrorKind::NonRecoverable => {
1688+
// Log and exit immediately for non-recoverable errors
1689+
let guidance = get_api_error_guidance(&error_msg);
1690+
error!(
1691+
"agent.api.non_recoverable error='{}' guidance='{}'",
1692+
error_msg, guidance
1693+
);
1694+
self.tui.add_log(format!("🛑 {}", guidance));
1695+
self.tui.add_log(
1696+
"❌ Agent stopping immediately due to non-recoverable API error".to_string()
1697+
);
1698+
return Err(anyhow!(
1699+
"Non-recoverable API error: {}. {}",
1700+
error_msg,
1701+
guidance
1702+
));
1703+
}
1704+
ApiErrorKind::Transient | ApiErrorKind::Unknown => {
1705+
// For transient errors, retry with exponential backoff
1706+
if attempt < retry_config.max_retries {
1707+
let delay = retry_config.delay_for_attempt(attempt);
1708+
let message = get_transient_error_message(&error_msg);
1709+
warn!(
1710+
"agent.api.transient attempt={}/{} delay_ms={} error='{}'",
1711+
attempt + 1,
1712+
retry_config.max_retries,
1713+
delay.as_millis(),
1714+
error_msg
1715+
);
1716+
self.tui.add_log(format!(
1717+
"{} (attempt {}/{}, waiting {}s)",
1718+
message,
1719+
attempt + 1,
1720+
retry_config.max_retries,
1721+
delay.as_secs()
1722+
));
1723+
tokio::time::sleep(delay).await;
1724+
last_error = Some(e);
1725+
continue;
1726+
} else {
1727+
// Exhausted all retries
1728+
error!(
1729+
"agent.api.retry_exhausted attempts={} error='{}'",
1730+
retry_config.max_retries + 1,
1731+
error_msg
1732+
);
1733+
self.tui.add_log(format!(
1734+
"❌ Exhausted {} retry attempts. Last error: {}",
1735+
retry_config.max_retries + 1,
1736+
error_msg
1737+
));
1738+
return Err(e);
1739+
}
1740+
}
1741+
}
16491742
}
16501743
}
16511744
}
1745+
// Should not reach here, but handle edge case
1746+
return Err(last_error.unwrap_or_else(|| anyhow!("Unknown error during retry")));
16521747
};
16531748

16541749
// Reset activity timer - we got an LLM response
@@ -3211,4 +3306,74 @@ mod tests {
32113306
assert!(get_api_error_guidance("account suspended").contains("Account"));
32123307
assert!(get_api_error_guidance("unknown error").contains("API"));
32133308
}
3309+
3310+
#[test]
3311+
fn test_retry_config_default() {
3312+
let config = RetryConfig::default();
3313+
assert_eq!(config.max_retries, 3);
3314+
assert_eq!(config.initial_delay_ms, 1000);
3315+
assert_eq!(config.max_delay_ms, 30000);
3316+
assert!((config.backoff_multiplier - 2.0).abs() < f64::EPSILON);
3317+
}
3318+
3319+
#[test]
3320+
fn test_retry_config_exponential_backoff() {
3321+
let config = RetryConfig::default();
3322+
3323+
// First attempt: 1000ms
3324+
let delay0 = config.delay_for_attempt(0);
3325+
assert_eq!(delay0.as_millis(), 1000);
3326+
3327+
// Second attempt: 2000ms (1000 * 2)
3328+
let delay1 = config.delay_for_attempt(1);
3329+
assert_eq!(delay1.as_millis(), 2000);
3330+
3331+
// Third attempt: 4000ms (1000 * 2^2)
3332+
let delay2 = config.delay_for_attempt(2);
3333+
assert_eq!(delay2.as_millis(), 4000);
3334+
3335+
// Fourth attempt: 8000ms (1000 * 2^3)
3336+
let delay3 = config.delay_for_attempt(3);
3337+
assert_eq!(delay3.as_millis(), 8000);
3338+
}
3339+
3340+
#[test]
3341+
fn test_retry_config_max_delay_cap() {
3342+
let config = RetryConfig {
3343+
max_retries: 10,
3344+
initial_delay_ms: 1000,
3345+
max_delay_ms: 5000,
3346+
backoff_multiplier: 2.0,
3347+
};
3348+
3349+
// After many retries, delay should cap at max_delay_ms
3350+
let delay10 = config.delay_for_attempt(10);
3351+
assert_eq!(delay10.as_millis(), 5000); // Capped at max
3352+
}
3353+
3354+
#[test]
3355+
fn test_get_transient_error_message_rate_limit() {
3356+
assert!(get_transient_error_message("rate limit exceeded").contains("Rate limit"));
3357+
assert!(get_transient_error_message("too many requests").contains("Rate limit"));
3358+
assert!(get_transient_error_message("Error 429: too many requests").contains("Rate limit"));
3359+
}
3360+
3361+
#[test]
3362+
fn test_get_transient_error_message_timeout() {
3363+
assert!(get_transient_error_message("request timeout").contains("timed out"));
3364+
assert!(get_transient_error_message("connection timeout").contains("timed out"));
3365+
}
3366+
3367+
#[test]
3368+
fn test_get_transient_error_message_network() {
3369+
assert!(get_transient_error_message("connection refused").contains("Network"));
3370+
assert!(get_transient_error_message("network error").contains("Network"));
3371+
assert!(get_transient_error_message("connection reset by peer").contains("Network"));
3372+
}
3373+
3374+
#[test]
3375+
fn test_get_transient_error_message_unknown() {
3376+
// Unknown transient errors should get a generic retry message
3377+
assert!(get_transient_error_message("some unknown error").contains("Transient"));
3378+
}
32143379
}

0 commit comments

Comments
 (0)