diff --git a/sdk/cliproxy/auth/conductor.go b/sdk/cliproxy/auth/conductor.go index 61afc5833f..d330eb1cde 100644 --- a/sdk/cliproxy/auth/conductor.go +++ b/sdk/cliproxy/auth/conductor.go @@ -1559,13 +1559,14 @@ func (m *Manager) Execute(ctx context.Context, providers []string, req cliproxye _, maxRetryCredentials, maxWait := m.retrySettings() var lastErr error + var failedAuthID string for attempt := 0; ; attempt++ { - resp, errExec := m.executeMixedOnce(ctx, normalized, req, opts, maxRetryCredentials) + resp, errExec := m.executeMixedOnce(ctx, normalized, req, opts, maxRetryCredentials, &failedAuthID) if errExec == nil { return resp, nil } lastErr = errExec - wait, shouldRetry := m.shouldRetryAfterError(errExec, attempt, normalized, req.Model, maxWait) + wait, shouldRetry := m.shouldRetryAfterError(errExec, attempt, normalized, req.Model, maxWait, failedAuthID) if !shouldRetry { break } @@ -1594,13 +1595,14 @@ func (m *Manager) ExecuteCount(ctx context.Context, providers []string, req clip _, maxRetryCredentials, maxWait := m.retrySettings() var lastErr error + var failedAuthID string for attempt := 0; ; attempt++ { - resp, errExec := m.executeCountMixedOnce(ctx, normalized, req, opts, maxRetryCredentials) + resp, errExec := m.executeCountMixedOnce(ctx, normalized, req, opts, maxRetryCredentials, &failedAuthID) if errExec == nil { return resp, nil } lastErr = errExec - wait, shouldRetry := m.shouldRetryAfterError(errExec, attempt, normalized, req.Model, maxWait) + wait, shouldRetry := m.shouldRetryAfterError(errExec, attempt, normalized, req.Model, maxWait, failedAuthID) if !shouldRetry { break } @@ -1625,13 +1627,14 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli _, maxRetryCredentials, maxWait := m.retrySettings() var lastErr error + var failedAuthID string for attempt := 0; ; attempt++ { - result, errStream := m.executeStreamMixedOnce(ctx, normalized, req, opts, maxRetryCredentials) + result, errStream := m.executeStreamMixedOnce(ctx, normalized, req, opts, maxRetryCredentials, &failedAuthID) if errStream == nil { return result, nil } lastErr = errStream - wait, shouldRetry := m.shouldRetryAfterError(errStream, attempt, normalized, req.Model, maxWait) + wait, shouldRetry := m.shouldRetryAfterError(errStream, attempt, normalized, req.Model, maxWait, failedAuthID) if !shouldRetry { break } @@ -1654,7 +1657,7 @@ func (m *Manager) ExecuteStream(ctx context.Context, providers []string, req cli return nil, &Error{Code: "auth_not_found", Message: "no auth available"} } -func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options, maxRetryCredentials int) (cliproxyexecutor.Response, error) { +func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options, maxRetryCredentials int, failedAuthID *string) (cliproxyexecutor.Response, error) { if len(providers) == 0 { return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"} } @@ -1709,6 +1712,9 @@ func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req result.Error.HTTPStatus = se.StatusCode() } m.MarkResult(execCtx, result) + if failedAuthID != nil { + *failedAuthID = auth.ID + } lastErr = errPrepare continue } @@ -1744,6 +1750,9 @@ func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req if isRequestInvalidError(authErr) { return cliproxyexecutor.Response{}, authErr } + if failedAuthID != nil { + *failedAuthID = auth.ID + } lastErr = authErr if homeMode { homeAuthCount++ @@ -1753,7 +1762,7 @@ func (m *Manager) executeMixedOnce(ctx context.Context, providers []string, req } } -func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options, maxRetryCredentials int) (cliproxyexecutor.Response, error) { +func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options, maxRetryCredentials int, failedAuthID *string) (cliproxyexecutor.Response, error) { if len(providers) == 0 { return cliproxyexecutor.Response{}, &Error{Code: "provider_not_found", Message: "no provider supplied"} } @@ -1808,6 +1817,9 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, result.Error.HTTPStatus = se.StatusCode() } m.MarkResult(execCtx, result) + if failedAuthID != nil { + *failedAuthID = auth.ID + } lastErr = errPrepare continue } @@ -1843,6 +1855,9 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, if isRequestInvalidError(authErr) { return cliproxyexecutor.Response{}, authErr } + if failedAuthID != nil { + *failedAuthID = auth.ID + } lastErr = authErr if homeMode { homeAuthCount++ @@ -1852,7 +1867,7 @@ func (m *Manager) executeCountMixedOnce(ctx context.Context, providers []string, } } -func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options, maxRetryCredentials int) (*cliproxyexecutor.StreamResult, error) { +func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string, req cliproxyexecutor.Request, opts cliproxyexecutor.Options, maxRetryCredentials int, failedAuthID *string) (*cliproxyexecutor.StreamResult, error) { if len(providers) == 0 { return nil, &Error{Code: "provider_not_found", Message: "no provider supplied"} } @@ -1905,6 +1920,9 @@ func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string result.Error.HTTPStatus = se.StatusCode() } m.MarkResult(execCtx, result) + if failedAuthID != nil { + *failedAuthID = auth.ID + } lastErr = errPrepare continue } @@ -1917,6 +1935,9 @@ func (m *Manager) executeStreamMixedOnce(ctx context.Context, providers []string if isRequestInvalidError(errStream) { return nil, errStream } + if failedAuthID != nil { + *failedAuthID = auth.ID + } lastErr = errStream if homeMode { homeAuthCount++ @@ -2569,11 +2590,47 @@ func (m *Manager) retryAllowed(attempt int, providers []string) bool { return false } -func (m *Manager) shouldRetryAfterError(err error, attempt int, providers []string, model string, maxWait time.Duration) (time.Duration, bool) { - if err == nil { - return 0, false +// eofRetryAllowed decides whether a statusless "unexpected EOF" may be retried. +// It honors the auth-file scoped request_retry override of the credential that +// actually failed (failedAuthID) so a credential that disabled retries is not +// retried just because another credential for the provider still allows it. +// When the failing auth is unknown it falls back to the provider-wide check. +func (m *Manager) eofRetryAllowed(attempt int, providers []string, failedAuthID string) bool { + if allowed, known := m.authRetryAllowed(attempt, failedAuthID); known { + return allowed } - if maxWait <= 0 { + return m.retryAllowed(attempt, providers) +} + +// authRetryAllowed reports whether the specific auth permits another attempt, +// honoring its auth-file scoped request_retry override. The second return value +// is false when the auth is unknown (empty ID or not registered). +func (m *Manager) authRetryAllowed(attempt int, authID string) (bool, bool) { + if m == nil || attempt < 0 { + return false, false + } + authID = strings.TrimSpace(authID) + if authID == "" { + return false, false + } + m.mu.RLock() + auth := m.auths[authID] + m.mu.RUnlock() + if auth == nil { + return false, false + } + effectiveRetry := int(m.requestRetry.Load()) + if override, ok := auth.RequestRetryOverride(); ok { + effectiveRetry = override + } + if effectiveRetry < 0 { + effectiveRetry = 0 + } + return attempt < effectiveRetry, true +} + +func (m *Manager) shouldRetryAfterError(err error, attempt int, providers []string, model string, maxWait time.Duration, failedAuthID string) (time.Duration, bool) { + if err == nil { return 0, false } status := statusCodeFromError(err) @@ -2583,6 +2640,31 @@ func (m *Manager) shouldRetryAfterError(err error, attempt int, providers []stri if isRequestInvalidError(err) { return 0, false } + // Treat transient, statusless upstream stream interruptions (e.g. + // "unexpected EOF") like a normal retryable API error instead of surfacing + // them to the client. These carry no HTTP status, so they would otherwise + // fall through the 429-only retry path below and abort the request. The + // status==0 guard is important: a status-bearing error (e.g. a 5xx whose + // body merely contains "unexpected EOF") has already had a cooldown applied + // by MarkResult, so it must fall through to the cooldown-aware path below + // rather than force an immediate retry that selection would reject with a + // model-cooldown error. eofRetryAllowed also honors the failed credential's + // auth-file scoped request_retry override. + // + // This is intentionally checked before the maxWait gate below: an EOF retry + // is immediate (wait=0) and needs no cooldown budget, so it must remain + // gated by the retry count alone and stay available even when + // max-retry-interval is 0 (which only caps how long we wait on cooled-down + // credentials). + if status == 0 && isUnexpectedEOFError(err) { + if !m.eofRetryAllowed(attempt, providers, failedAuthID) { + return 0, false + } + return 0, true + } + if maxWait <= 0 { + return 0, false + } wait, found := m.closestCooldownWait(providers, model, attempt) if found { if wait > maxWait { @@ -2969,6 +3051,20 @@ func errorString(err error) string { return err.Error() } +// isUnexpectedEOFError reports whether err represents an "unexpected EOF" +// surfaced when an upstream connection or stream is cut off mid-response. +// These are transient connection failures rather than genuine API errors, so +// callers should retry them instead of returning them to the client. +func isUnexpectedEOFError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + return strings.Contains(strings.ToLower(err.Error()), "unexpected eof") +} + func statusCodeFromError(err error) int { if err == nil { return 0 diff --git a/sdk/cliproxy/auth/conductor_overrides_test.go b/sdk/cliproxy/auth/conductor_overrides_test.go index 5acd331e1f..069594822d 100644 --- a/sdk/cliproxy/auth/conductor_overrides_test.go +++ b/sdk/cliproxy/auth/conductor_overrides_test.go @@ -2,6 +2,8 @@ package auth import ( "context" + "fmt" + "io" "net/http" "sync" "testing" @@ -41,7 +43,7 @@ func TestManager_ShouldRetryAfterError_RespectsAuthRequestRetryOverride(t *testi } _, _, maxWait := m.retrySettings() - wait, shouldRetry := m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 0, []string{"claude"}, model, maxWait) + wait, shouldRetry := m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 0, []string{"claude"}, model, maxWait, "") if shouldRetry { t.Fatalf("expected shouldRetry=false for request_retry=0, got true (wait=%v)", wait) } @@ -51,7 +53,7 @@ func TestManager_ShouldRetryAfterError_RespectsAuthRequestRetryOverride(t *testi t.Fatalf("update auth: %v", errUpdate) } - wait, shouldRetry = m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 0, []string{"claude"}, model, maxWait) + wait, shouldRetry = m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 0, []string{"claude"}, model, maxWait, "") if !shouldRetry { t.Fatalf("expected shouldRetry=true for request_retry=1, got false") } @@ -59,7 +61,7 @@ func TestManager_ShouldRetryAfterError_RespectsAuthRequestRetryOverride(t *testi t.Fatalf("expected wait > 0, got %v", wait) } - _, shouldRetry = m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 1, []string{"claude"}, model, maxWait) + _, shouldRetry = m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 1, []string{"claude"}, model, maxWait, "") if shouldRetry { t.Fatalf("expected shouldRetry=false on attempt=1 for request_retry=1, got true") } @@ -99,7 +101,7 @@ func TestManager_ShouldRetryAfterError_UsesOAuthModelAliasForCooldown(t *testing } _, _, maxWait := m.retrySettings() - wait, shouldRetry := m.shouldRetryAfterError(&Error{HTTPStatus: 429, Message: "quota"}, 0, []string{"kimi"}, routeModel, maxWait) + wait, shouldRetry := m.shouldRetryAfterError(&Error{HTTPStatus: 429, Message: "quota"}, 0, []string{"kimi"}, routeModel, maxWait, "") if !shouldRetry { t.Fatalf("expected shouldRetry=true, got false (wait=%v)", wait) } @@ -108,6 +110,271 @@ func TestManager_ShouldRetryAfterError_UsesOAuthModelAliasForCooldown(t *testing } } +func TestManager_ShouldRetryAfterError_RetriesUnexpectedEOF(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(3, 30*time.Second, 0) + + model := "test-model" + auth := &Auth{ID: "auth-1", Provider: "claude"} + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + _, _, maxWait := m.retrySettings() + + // A statusless "unexpected EOF" error (as produced by a truncated upstream + // stream) must be retried immediately rather than surfaced to the client. + wait, shouldRetry := m.shouldRetryAfterError(&Error{Message: "unexpected EOF"}, 0, []string{"claude"}, model, maxWait, "") + if !shouldRetry { + t.Fatalf("expected shouldRetry=true for unexpected EOF, got false") + } + if wait != 0 { + t.Fatalf("expected immediate retry (wait=0), got %v", wait) + } + + // io.ErrUnexpectedEOF (possibly wrapped) is detected as well. + wrapped := fmt.Errorf("read stream: %w", io.ErrUnexpectedEOF) + if _, shouldRetry = m.shouldRetryAfterError(wrapped, 0, []string{"claude"}, model, maxWait, ""); !shouldRetry { + t.Fatalf("expected shouldRetry=true for wrapped io.ErrUnexpectedEOF, got false") + } + + // Retries stop once the configured request-retry count is exhausted. + if _, shouldRetry = m.shouldRetryAfterError(&Error{Message: "unexpected EOF"}, 3, []string{"claude"}, model, maxWait, ""); shouldRetry { + t.Fatalf("expected shouldRetry=false on attempt=3 for request_retry=3, got true") + } +} + +func TestManager_ShouldRetryAfterError_StatusBearingEOFUsesCooldownPath(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(3, 30*time.Second, 0) + + model := "test-model" + // MarkResult applies a one-minute cooldown for transient 5xx responses. + next := time.Now().Add(time.Minute) + auth := &Auth{ + ID: "auth-1", + Provider: "claude", + ModelStates: map[string]*ModelState{ + model: { + Unavailable: true, + Status: StatusError, + NextRetryAfter: next, + }, + }, + } + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + _, _, maxWait := m.retrySettings() + // A 5xx whose body merely contains "unexpected EOF" must NOT take the + // immediate fast path; with a single cooled-down credential and a cooldown + // exceeding max-retry-interval, the cooldown-aware path declines the retry. + statusErr := &Error{HTTPStatus: 500, Message: "internal error: unexpected EOF"} + if wait, shouldRetry := m.shouldRetryAfterError(statusErr, 0, []string{"claude"}, model, maxWait, ""); shouldRetry { + t.Fatalf("expected shouldRetry=false for status-bearing EOF beyond cooldown, got true (wait=%v)", wait) + } +} + +func TestManager_ShouldRetryAfterError_UnexpectedEOFRespectsRetryDisabled(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(0, 30*time.Second, 0) + + auth := &Auth{ID: "auth-1", Provider: "claude"} + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + _, _, maxWait := m.retrySettings() + if _, shouldRetry := m.shouldRetryAfterError(&Error{Message: "unexpected EOF"}, 0, []string{"claude"}, "test-model", maxWait, ""); shouldRetry { + t.Fatalf("expected shouldRetry=false when request-retry=0, got true") + } +} + +func TestManager_ShouldRetryAfterError_UnexpectedEOFRespectsAuthRequestRetryOverride(t *testing.T) { + m := NewManager(nil, nil, nil) + m.SetRetryConfig(3, 30*time.Second, 0) + + model := "test-model" + // The credential that fails has retries disabled via its auth-file override. + disabled := &Auth{ + ID: "auth-disabled", + Provider: "claude", + Metadata: map[string]any{"request_retry": float64(0)}, + } + // A sibling credential for the same provider still allows retries. + enabled := &Auth{ID: "auth-enabled", Provider: "claude"} + for _, a := range []*Auth{disabled, enabled} { + if _, errRegister := m.Register(context.Background(), a); errRegister != nil { + t.Fatalf("register %s: %v", a.ID, errRegister) + } + } + + _, _, maxWait := m.retrySettings() + eof := &Error{Message: "unexpected EOF"} + + // When the failing credential disabled retries, its statusless EOF must not + // be retried even though a sibling credential still allows retries. + if _, shouldRetry := m.shouldRetryAfterError(eof, 0, []string{"claude"}, model, maxWait, "auth-disabled"); shouldRetry { + t.Fatalf("expected shouldRetry=false for EOF on request_retry=0 credential, got true") + } + + // The retry-enabled credential still retries its EOF. + if _, shouldRetry := m.shouldRetryAfterError(eof, 0, []string{"claude"}, model, maxWait, "auth-enabled"); !shouldRetry { + t.Fatalf("expected shouldRetry=true for EOF on retry-enabled credential, got false") + } + + // An unknown failing auth falls back to the provider-wide retry check. + if _, shouldRetry := m.shouldRetryAfterError(eof, 0, []string{"claude"}, model, maxWait, ""); !shouldRetry { + t.Fatalf("expected shouldRetry=true for EOF with unknown auth fallback, got false") + } +} + +// TestManager_ShouldRetryAfterError_UnexpectedEOFRetriesWhenMaxRetryIntervalZero +// verifies that the immediate statusless EOF retry stays gated by the retry +// count alone and is not suppressed by max-retry-interval=0. An EOF retry needs +// no cooldown wait budget, so it must remain available even when waiting on +// cooled-down credentials is disabled (maxWait=0). Status-bearing errors that +// would need a wait budget are still correctly gated off. +func TestManager_ShouldRetryAfterError_UnexpectedEOFRetriesWhenMaxRetryIntervalZero(t *testing.T) { + m := NewManager(nil, nil, nil) + // request-retry: 3, max-retry-interval: 0 (a documented, valid config). + m.SetRetryConfig(3, 0, 0) + + model := "test-model" + auth := &Auth{ID: "auth-1", Provider: "claude"} + if _, errRegister := m.Register(context.Background(), auth); errRegister != nil { + t.Fatalf("register auth: %v", errRegister) + } + + _, _, maxWait := m.retrySettings() + if maxWait != 0 { + t.Fatalf("expected maxWait=0 from max-retry-interval=0, got %v", maxWait) + } + + eof := &Error{Message: "unexpected EOF"} + if wait, shouldRetry := m.shouldRetryAfterError(eof, 0, []string{"claude"}, model, maxWait, ""); !shouldRetry || wait != 0 { + t.Fatalf("expected immediate EOF retry (wait=0, shouldRetry=true) with max-retry-interval=0, got wait=%v shouldRetry=%v", wait, shouldRetry) + } + + // Once the retry count is exhausted the EOF stops being retried. + if _, shouldRetry := m.shouldRetryAfterError(eof, 3, []string{"claude"}, model, maxWait, ""); shouldRetry { + t.Fatalf("expected no EOF retry once request-retry is exhausted, got true") + } + + // A status-bearing error that needs a cooldown wait budget remains gated by + // max-retry-interval=0. + if _, shouldRetry := m.shouldRetryAfterError(&Error{HTTPStatus: 500, Message: "boom"}, 0, []string{"claude"}, model, maxWait, ""); shouldRetry { + t.Fatalf("expected no retry for status-bearing error with max-retry-interval=0, got true") + } +} + +// TestManager_Execute_UnexpectedEOFHonorsFailingAuthRequestRetryOverride drives +// the full Execute/ExecuteStream retry loop to verify that the credential which +// actually produced the statusless "unexpected EOF" is the one whose +// request_retry override gates the retry. The failing credential disables +// retries (request_retry=0) while a sibling that still allows retries is parked +// on cooldown so it is never selected; the sibling's presence makes the +// provider-wide retry check return true, so an immediate EOF retry here would +// only be suppressed if the loop tracks the failing credential correctly. +func TestManager_Execute_UnexpectedEOFHonorsFailingAuthRequestRetryOverride(t *testing.T) { + model := "test-model" + eofErr := &Error{Message: "unexpected EOF"} + + testCases := []struct { + name string + calls func(*authFallbackExecutor) []string + run func(*Manager) error + }{ + { + name: "execute", + calls: (*authFallbackExecutor).ExecuteCalls, + run: func(m *Manager) error { + _, err := m.Execute(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}) + return err + }, + }, + { + name: "execute_stream", + calls: (*authFallbackExecutor).StreamCalls, + run: func(m *Manager) error { + // A statusless bootstrap EOF is surfaced through the stream's + // chunks (returned error is nil), so drain and report it. + result, err := m.ExecuteStream(context.Background(), []string{"claude"}, cliproxyexecutor.Request{Model: model}, cliproxyexecutor.Options{}) + if err != nil { + return err + } + if result == nil { + return nil + } + for chunk := range result.Chunks { + if chunk.Err != nil { + return chunk.Err + } + } + return nil + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + m := NewManager(nil, nil, nil) + // Provider-wide retries are enabled; only the failing credential's + // request_retry=0 override must stop the EOF from being retried. + m.SetRetryConfig(3, 30*time.Second, 0) + + baseID := uuid.NewString() + failing := &Auth{ + ID: baseID + "-failing", + Provider: "claude", + Metadata: map[string]any{"request_retry": float64(0)}, + } + sibling := &Auth{ + ID: baseID + "-sibling", + Provider: "claude", + ModelStates: map[string]*ModelState{ + model: {Unavailable: true, Status: StatusError, NextRetryAfter: time.Now().Add(time.Hour)}, + }, + } + + executor := &authFallbackExecutor{ + id: "claude", + executeErrors: map[string]error{failing.ID: eofErr}, + streamFirstErrors: map[string]error{failing.ID: eofErr}, + } + m.RegisterExecutor(executor) + + reg := registry.GetGlobalRegistry() + reg.RegisterClient(failing.ID, "claude", []*registry.ModelInfo{{ID: model}}) + reg.RegisterClient(sibling.ID, "claude", []*registry.ModelInfo{{ID: model}}) + t.Cleanup(func() { + reg.UnregisterClient(failing.ID) + reg.UnregisterClient(sibling.ID) + }) + + for _, a := range []*Auth{failing, sibling} { + if _, errRegister := m.Register(context.Background(), a); errRegister != nil { + t.Fatalf("register %s: %v", a.ID, errRegister) + } + } + + if errRun := tc.run(m); errRun == nil { + t.Fatalf("expected error from unexpected EOF, got nil") + } + + calls := tc.calls(executor) + if len(calls) != 1 { + t.Fatalf("expected the failing credential to be tried exactly once (no EOF retry under request_retry=0), got %d calls: %v", len(calls), calls) + } + if calls[0] != failing.ID { + t.Fatalf("expected only the failing credential %q to be tried, got %v", failing.ID, calls) + } + }) + } +} + type credentialRetryLimitExecutor struct { id string