Skip to content

Commit 197c570

Browse files
Wei-ShawWei-Shaw
authored
Merge pull request Wei-Shaw#723 from zqq-nuli/fix/oauth-401-temp-unschedulable
fix: OAuth 401 不再永久锁死账号，改用临时不可调度实现自动恢复
2 parents 0fe09f1 + ec6bcfe commit 197c570

7 files changed

Lines changed: 175 additions & 49 deletions

File tree

‎backend/cmd/server/wire_gen.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎backend/internal/config/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -873,6 +873,7 @@ type DefaultConfig struct {
873873

874874
type RateLimitConfig struct {
875875
OverloadCooldownMinutes int `mapstructure:"overload_cooldown_minutes"` // 529过载冷却时间(分钟)
876+
OAuth401CooldownMinutes int `mapstructure:"oauth_401_cooldown_minutes"` // OAuth 401临时不可调度冷却(分钟)
876877
}
877878

878879
// APIKeyAuthCacheConfig API Key 认证缓存配置
@@ -1260,6 +1261,7 @@ func setDefaults() {
12601261

12611262
// RateLimit
12621263
viper.SetDefault("rate_limit.overload_cooldown_minutes", 10)
1264+
viper.SetDefault("rate_limit.oauth_401_cooldown_minutes", 10)
12631265

12641266
// Pricing - 从 model-price-repo 同步模型定价和上下文窗口数据（固定到 commit，避免分支漂移）
12651267
viper.SetDefault("pricing.remote_url", "https://raw.githubusercontent.com/Wei-Shaw/model-price-repo/c7947e9871687e664180bc971d4837f1fc2784a9/model_prices_and_context_window.json")

‎backend/internal/service/ratelimit_service.go‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -146,13 +146,29 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc
146146
} else {
147147
slog.Info("oauth_401_force_refresh_set", "account_id", account.ID, "platform", account.Platform)
148148
}
149+
// 3. 临时不可调度，替代 SetError（保持 status=active 让刷新服务能拾取）
150+
msg := "Authentication failed (401): invalid or expired credentials"
151+
if upstreamMsg != "" {
152+
msg = "OAuth 401: " + upstreamMsg
153+
}
154+
cooldownMinutes := s.cfg.RateLimit.OAuth401CooldownMinutes
155+
if cooldownMinutes <= 0 {
156+
cooldownMinutes = 10
157+
}
158+
until := time.Now().Add(time.Duration(cooldownMinutes) * time.Minute)
159+
if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, msg); err != nil {
160+
slog.Warn("oauth_401_set_temp_unschedulable_failed", "account_id", account.ID, "error", err)
161+
}
162+
shouldDisable = true
163+
} else {
164+
// 非 OAuth 账号（APIKey）：保持原有 SetError 行为
165+
msg := "Authentication failed (401): invalid or expired credentials"
166+
if upstreamMsg != "" {
167+
msg = "Authentication failed (401): " + upstreamMsg
168+
}
169+
s.handleAuthError(ctx, account, msg)
170+
shouldDisable = true
149171
}
150-
msg := "Authentication failed (401): invalid or expired credentials"
151-
if upstreamMsg != "" {
152-
msg = "Authentication failed (401): " + upstreamMsg
153-
}
154-
s.handleAuthError(ctx, account, msg)
155-
shouldDisable = true
156172
case 402:
157173
// 支付要求：余额不足或计费问题，停止调度
158174
msg := "Payment required (402): insufficient balance or billing issue"

‎backend/internal/service/ratelimit_service_401_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (r *tokenCacheInvalidatorRecorder) InvalidateToken(ctx context.Context, acc
4141
return r.err
4242
}
4343

44-
func TestRateLimitService_HandleUpstreamError_OAuth401MarksError(t *testing.T) {
44+
func TestRateLimitService_HandleUpstreamError_OAuth401SetsTempUnschedulable(t *testing.T) {
4545
tests := []struct {
4646
name string
4747
platform string
@@ -76,9 +76,8 @@ func TestRateLimitService_HandleUpstreamError_OAuth401MarksError(t *testing.T) {
7676
shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
7777

7878
require.True(t, shouldDisable)
79-
require.Equal(t, 1, repo.setErrorCalls)
80-
require.Equal(t, 0, repo.tempCalls)
81-
require.Contains(t, repo.lastErrorMsg, "Authentication failed (401)")
79+
require.Equal(t, 0, repo.setErrorCalls)
80+
require.Equal(t, 1, repo.tempCalls)
8281
require.Len(t, invalidator.accounts, 1)
8382
})
8483
}
@@ -98,7 +97,8 @@ func TestRateLimitService_HandleUpstreamError_OAuth401InvalidatorError(t *testin
9897
shouldDisable := service.HandleUpstreamError(context.Background(), account, 401, http.Header{}, []byte("unauthorized"))
9998

10099
require.True(t, shouldDisable)
101-
require.Equal(t, 1, repo.setErrorCalls)
100+
require.Equal(t, 0, repo.setErrorCalls)
101+
require.Equal(t, 1, repo.tempCalls)
102102
require.Len(t, invalidator.accounts, 1)
103103
}
104104

‎backend/internal/service/token_refresh_service.go‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ type TokenRefreshService struct {
1818
refreshers []TokenRefresher
1919
cfg *config.TokenRefreshConfig
2020
cacheInvalidator TokenCacheInvalidator
21-
schedulerCache SchedulerCache // 用于同步更新调度器缓存，解决 token 刷新后缓存不一致问题
21+
schedulerCache SchedulerCache // 用于同步更新调度器缓存，解决 token 刷新后缓存不一致问题
22+
tempUnschedCache TempUnschedCache // 用于清除 Redis 中的临时不可调度缓存
2223

2324
stopCh chan struct{}
2425
wg sync.WaitGroup
@@ -34,12 +35,14 @@ func NewTokenRefreshService(
3435
cacheInvalidator TokenCacheInvalidator,
3536
schedulerCache SchedulerCache,
3637
cfg *config.Config,
38+
tempUnschedCache TempUnschedCache,
3739
) *TokenRefreshService {
3840
s := &TokenRefreshService{
3941
accountRepo: accountRepo,
4042
cfg: &cfg.TokenRefresh,
4143
cacheInvalidator: cacheInvalidator,
4244
schedulerCache: schedulerCache,
45+
tempUnschedCache: tempUnschedCache,
4346
stopCh: make(chan struct{}),
4447
}
4548

@@ -231,6 +234,26 @@ func (s *TokenRefreshService) refreshWithRetry(ctx context.Context, account *Acc
231234
slog.Info("token_refresh.cleared_missing_project_id_error", "account_id", account.ID)
232235
}
233236
}
237+
// 刷新成功后清除临时不可调度状态（处理 OAuth 401 恢复场景）
238+
if account.TempUnschedulableUntil != nil && time.Now().Before(*account.TempUnschedulableUntil) {
239+
if clearErr := s.accountRepo.ClearTempUnschedulable(ctx, account.ID); clearErr != nil {
240+
slog.Warn("token_refresh.clear_temp_unschedulable_failed",
241+
"account_id", account.ID,
242+
"error", clearErr,
243+
)
244+
} else {
245+
slog.Info("token_refresh.cleared_temp_unschedulable", "account_id", account.ID)
246+
}
247+
// 同步清除 Redis 缓存，避免调度器读到过期的临时不可调度状态
248+
if s.tempUnschedCache != nil {
249+
if clearErr := s.tempUnschedCache.DeleteTempUnsched(ctx, account.ID); clearErr != nil {
250+
slog.Warn("token_refresh.clear_temp_unsched_cache_failed",
251+
"account_id", account.ID,
252+
"error", clearErr,
253+
)
254+
}
255+
}
256+
}
234257
// 对所有 OAuth 账号调用缓存失效（InvalidateToken 内部根据平台判断是否需要处理）
235258
if s.cacheInvalidator != nil && account.Type == AccountTypeOAuth {
236259
if err := s.cacheInvalidator.InvalidateToken(ctx, account); err != nil {
@@ -257,8 +280,8 @@ func (s *TokenRefreshService) refreshWithRetry(ctx context.Context, account *Acc
257280
return nil
258281
}
259282

260-
// Antigravity 账户：不可重试错误直接标记 error 状态并返回
261-
if account.Platform == PlatformAntigravity && isNonRetryableRefreshError(err) {
283+
// 不可重试错误（invalid_grant/invalid_client 等）直接标记 error 状态并返回
284+
if isNonRetryableRefreshError(err) {
262285
errorMsg := fmt.Sprintf("Token refresh failed (non-retryable): %v", err)
263286
if setErr := s.accountRepo.SetError(ctx, account.ID, errorMsg); setErr != nil {
264287
slog.Error("token_refresh.set_error_status_failed",
@@ -285,23 +308,13 @@ func (s *TokenRefreshService) refreshWithRetry(ctx context.Context, account *Acc
285308
}
286309
}
287310

288-
// Antigravity 账户：其他错误仅记录日志，不标记 error（可能是临时网络问题）
289-
// 其他平台账户：重试失败后标记 error
290-
if account.Platform == PlatformAntigravity {
291-
slog.Warn("token_refresh.retry_exhausted_antigravity",
292-
"account_id", account.ID,
293-
"max_retries", s.cfg.MaxRetries,
294-
"error", lastErr,
295-
)
296-
} else {
297-
errorMsg := fmt.Sprintf("Token refresh failed after %d retries: %v", s.cfg.MaxRetries, lastErr)
298-
if err := s.accountRepo.SetError(ctx, account.ID, errorMsg); err != nil {
299-
slog.Error("token_refresh.set_error_status_failed",
300-
"account_id", account.ID,
301-
"error", err,
302-
)
303-
}
304-
}
311+
// 可重试错误耗尽：仅记录日志，不标记 error（可能是临时网络问题，下个周期继续重试）
312+
slog.Warn("token_refresh.retry_exhausted",
313+
"account_id", account.ID,
314+
"platform", account.Platform,
315+
"max_retries", s.cfg.MaxRetries,
316+
"error", lastErr,
317+
)
305318

306319
return lastErr
307320
}

0 commit comments

Comments
 (0)