Skip to content

Commit c5de131

Browse files
committed
Fix retry error aggregation
1 parent d1d3557 commit c5de131

1 file changed

Lines changed: 102 additions & 16 deletions

File tree

src/client.rs

Lines changed: 102 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -60,16 +60,27 @@ macro_rules! impl_inner_call {
6060
},
6161
Err(e) => {
6262
let failed_attempts = errors.len() + 1;
63-
64-
if retries_exhausted(failed_attempts, $self.config.retry()) {
65-
warn!("call '{}' failed after {} attempts", stringify!($name), failed_attempts);
66-
return Err(Error::AllAttemptsErrored(errors));
63+
let configured_retries = $self.config.retry();
64+
65+
warn!(
66+
"call '{}' failed with {}, retry: {}/{}",
67+
stringify!($name),
68+
e,
69+
failed_attempts,
70+
configured_retries
71+
);
72+
73+
if let Err(err) =
74+
record_attempt_error(&mut errors, e, configured_retries)
75+
{
76+
warn!(
77+
"call '{}' failed after {} attempts",
78+
stringify!($name),
79+
failed_attempts
80+
);
81+
return Err(err);
6782
}
6883

69-
warn!("call '{}' failed with {}, retry: {}/{}", stringify!($name), e, failed_attempts, $self.config.retry());
70-
71-
errors.push(e);
72-
7384
// Only one thread will try to recreate the client getting the write lock,
7485
// other eventual threads will get Err and will block at the beginning of
7586
// previous loop when trying to read()
@@ -84,15 +95,24 @@ macro_rules! impl_inner_call {
8495
},
8596
Err(e) => {
8697
let failed_attempts = errors.len() + 1;
87-
88-
if retries_exhausted(failed_attempts, $self.config.retry()) {
89-
warn!("re-creating client failed after {} attempts", failed_attempts);
90-
return Err(Error::AllAttemptsErrored(errors));
98+
let configured_retries = $self.config.retry();
99+
100+
warn!(
101+
"re-creating client failed with {}, retry: {}/{}",
102+
e,
103+
failed_attempts,
104+
configured_retries
105+
);
106+
107+
if let Err(err) =
108+
record_attempt_error(&mut errors, e, configured_retries)
109+
{
110+
warn!(
111+
"re-creating client failed after {} attempts",
112+
failed_attempts
113+
);
114+
return Err(err);
91115
}
92-
93-
warn!("re-creating client failed with {}, retry: {}/{}", e, failed_attempts, $self.config.retry());
94-
95-
errors.push(e);
96116
}
97117
}
98118
}
@@ -103,6 +123,22 @@ macro_rules! impl_inner_call {
103123
}
104124
}
105125

126+
fn record_attempt_error(
127+
errors: &mut Vec<Error>,
128+
error: Error,
129+
configured_retries: u8,
130+
) -> Result<usize, Error> {
131+
let failed_attempts = errors.len() + 1;
132+
133+
errors.push(error);
134+
135+
if retries_exhausted(failed_attempts, configured_retries) {
136+
Err(Error::AllAttemptsErrored(std::mem::take(errors)))
137+
} else {
138+
Ok(failed_attempts)
139+
}
140+
}
141+
106142
fn retries_exhausted(failed_attempts: usize, configured_retries: u8) -> bool {
107143
match u8::try_from(failed_attempts) {
108144
Ok(failed_attempts) => failed_attempts > configured_retries,
@@ -464,4 +500,54 @@ mod tests {
464500

465501
assert!(!exhausted)
466502
}
503+
504+
#[test]
505+
fn exhausted_attempt_includes_current_error() {
506+
let mut errors = Vec::new();
507+
508+
let err = record_attempt_error(
509+
&mut errors,
510+
Error::Message("current".to_string()),
511+
0,
512+
)
513+
.unwrap_err();
514+
515+
let Error::AllAttemptsErrored(attempts) = err else {
516+
panic!("expected AllAttemptsErrored");
517+
};
518+
519+
assert_eq!(attempts.len(), 1);
520+
assert!(
521+
matches!(&attempts[0], Error::Message(message) if message == "current")
522+
);
523+
assert!(errors.is_empty());
524+
}
525+
526+
#[test]
527+
fn retry_errors_accumulate_until_exhausted() {
528+
let mut errors = Vec::new();
529+
530+
assert_eq!(
531+
record_attempt_error(&mut errors, Error::Message("first".to_string()), 1).unwrap(),
532+
1
533+
);
534+
535+
let err = record_attempt_error(
536+
&mut errors,
537+
Error::Message("second".to_string()),
538+
1,
539+
)
540+
.unwrap_err();
541+
542+
let Error::AllAttemptsErrored(attempts) = err else {
543+
panic!("expected AllAttemptsErrored");
544+
};
545+
546+
assert_eq!(attempts.len(), 2);
547+
assert!(matches!(&attempts[0], Error::Message(message) if message == "first"));
548+
assert!(
549+
matches!(&attempts[1], Error::Message(message) if message == "second")
550+
);
551+
assert!(errors.is_empty());
552+
}
467553
}

0 commit comments

Comments
 (0)