feat(oauth): implement token refresh reliability with exponential backoff (smart-mcp-proxy#23)

Implements the OAuth token refresh reliability feature per Spec 023.

Key changes:
- Add RefreshState type for tracking refresh status (idle/scheduled/retrying/failed)
- Implement exponential backoff for refresh failures (10s → 20s → 40s → ... → 5m cap)
- Add rate limiting (min 10s between attempts per server)
- Detect expired tokens at startup and attempt immediate refresh
- Continue retries until token expiration (unlimited retries per FR-009)
- Classify errors as permanent (invalid_grant) vs retryable (network)
- Extend health calculator to show refresh state in health status
- Add OAuth refresh metrics (mcpproxy_oauth_refresh_total, duration)
- Fix misleading logging: rename LogTokenRefreshSuccess to LogClientConnectionSuccess

User Stories Implemented:
- US1: Survive laptop sleep/wake - auto-reconnect using stored refresh tokens
- US2: Proactive token refresh at 80% lifetime with exponential backoff
- US3: Clear refresh failure feedback in health status

Spec: smart-mcp-proxy#253

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: technicalpickles

internal/health/calculator.go

@@ -8,6 +8,21 @@ import (
 	"mcpproxy-go/internal/contracts"
 )
 
+// RefreshState represents the current state of token refresh for health reporting.
+// Mirrors oauth.RefreshState for decoupling.
+type RefreshState int
+
+const (
+	// RefreshStateIdle means no refresh is pending or in progress.
+	RefreshStateIdle RefreshState = iota
+	// RefreshStateScheduled means a proactive refresh is scheduled at 80% lifetime.
+	RefreshStateScheduled
+	// RefreshStateRetrying means refresh failed and is retrying with exponential backoff.
+	RefreshStateRetrying
+	// RefreshStateFailed means refresh permanently failed (e.g., invalid_grant).
+	RefreshStateFailed
+)
+
 // HealthCalculatorInput contains all fields needed to calculate health status.
 // This struct normalizes data from different sources (StateView, storage, config).
 type HealthCalculatorInput struct {
@@ -36,6 +51,12 @@ type HealthCalculatorInput struct {
 
 	// Tool info
 	ToolCount int
+
+	// Refresh state (for health status integration - Spec 023)
+	RefreshState       RefreshState // Current refresh state from RefreshManager
+	RefreshRetryCount  int          // Number of retry attempts
+	RefreshLastError   string       // Human-readable error message
+	RefreshNextAttempt *time.Time   // When next retry will occur
 }
 
 // HealthCalculatorConfig contains configurable thresholds for health calculation.
@@ -219,7 +240,35 @@ func CalculateHealth(input HealthCalculatorInput, cfg *HealthCalculatorConfig) *
 		}
 	}
 
-	// 6. Healthy state - connected with valid authentication (if required)
+	// 6. Refresh state checks (Spec 023)
+	// Check if refresh is in a degraded or failed state
+	switch input.RefreshState {
+	case RefreshStateRetrying:
+		// Refresh failed but retrying - degraded status
+		detail := formatRefreshRetryDetail(input.RefreshRetryCount, input.RefreshNextAttempt, input.RefreshLastError)
+		return &contracts.HealthStatus{
+			Level:      LevelDegraded,
+			AdminState: StateEnabled,
+			Summary:    "Token refresh pending",
+			Detail:     detail,
+			Action:     ActionViewLogs,
+		}
+	case RefreshStateFailed:
+		// Refresh permanently failed - unhealthy status
+		detail := "Re-authentication required"
+		if input.RefreshLastError != "" {
+			detail = fmt.Sprintf("Re-authentication required: %s", input.RefreshLastError)
+		}
+		return &contracts.HealthStatus{
+			Level:      LevelUnhealthy,
+			AdminState: StateEnabled,
+			Summary:    "Refresh token expired",
+			Detail:     detail,
+			Action:     ActionLogin,
+		}
+	}
+
+	// 7. Healthy state - connected with valid authentication (if required)
 	return &contracts.HealthStatus{
 		Level:      LevelHealthy,
 		AdminState: StateEnabled,
@@ -302,6 +351,30 @@ func formatExpiringTokenSummary(timeUntilExpiry time.Duration) string {
 	return fmt.Sprintf("Token expiring in %dh", hours)
 }
 
+// formatRefreshRetryDetail formats the detail message for a refresh retry state.
+func formatRefreshRetryDetail(retryCount int, nextAttempt *time.Time, lastError string) string {
+	var detail string
+
+	// Start with retry count and next attempt time
+	if nextAttempt != nil && !nextAttempt.IsZero() {
+		detail = fmt.Sprintf("Refresh retry %d scheduled for %s", retryCount, nextAttempt.Format(time.RFC3339))
+	} else {
+		detail = fmt.Sprintf("Refresh retry %d pending", retryCount)
+	}
+
+	// Add last error if available
+	if lastError != "" {
+		// Truncate error if too long
+		errorMsg := lastError
+		if len(errorMsg) > 100 {
+			errorMsg = errorMsg[:97] + "..."
+		}
+		detail = fmt.Sprintf("%s: %s", detail, errorMsg)
+	}
+
+	return detail
+}
+
 // containsIgnoreCase checks if s contains substr, ignoring case.
 func containsIgnoreCase(s, substr string) bool {
 	return len(s) >= len(substr) &&

internal/oauth/logging.go

@@ -159,29 +159,78 @@ func LogTokenMetadata(logger *zap.Logger, metadata TokenMetadata) {
 	)
 }
 
-// LogTokenRefreshAttempt logs a token refresh attempt.
-func LogTokenRefreshAttempt(logger *zap.Logger, attempt int, maxAttempts int) {
-	logger.Info("Attempting OAuth token refresh",
+// LogClientConnectionAttempt logs a client connection attempt (not an actual token refresh).
+// Note: This is called when retrying client.Start(), which may trigger automatic
+// token refresh internally by mcp-go, but we cannot observe whether refresh actually occurred.
+func LogClientConnectionAttempt(logger *zap.Logger, attempt int, maxAttempts int) {
+	logger.Info("OAuth client connection attempt",
 		zap.Int("attempt", attempt),
 		zap.Int("max_attempts", maxAttempts),
 	)
 }
 
-// LogTokenRefreshSuccess logs a successful token refresh.
-func LogTokenRefreshSuccess(logger *zap.Logger, duration time.Duration) {
-	logger.Info("OAuth token refresh successful",
+// LogClientConnectionSuccess logs a successful client connection.
+// Note: This does NOT mean a token refresh occurred - it means the client connected.
+// The mcp-go library may have used a cached token or performed automatic refresh internally.
+func LogClientConnectionSuccess(logger *zap.Logger, duration time.Duration) {
+	logger.Info("OAuth client connection successful",
 		zap.Duration("duration", duration),
 	)
 }
 
-// LogTokenRefreshFailure logs a failed token refresh attempt.
-func LogTokenRefreshFailure(logger *zap.Logger, attempt int, err error) {
-	logger.Warn("OAuth token refresh failed",
+// LogClientConnectionFailure logs a failed client connection attempt.
+func LogClientConnectionFailure(logger *zap.Logger, attempt int, err error) {
+	logger.Warn("OAuth client connection failed",
 		zap.Int("attempt", attempt),
 		zap.Error(err),
 	)
 }
 
+// Deprecated: Use LogClientConnectionAttempt instead.
+// LogTokenRefreshAttempt is kept for backward compatibility but is misleading.
+func LogTokenRefreshAttempt(logger *zap.Logger, attempt int, maxAttempts int) {
+	LogClientConnectionAttempt(logger, attempt, maxAttempts)
+}
+
+// Deprecated: Use LogClientConnectionSuccess instead.
+// LogTokenRefreshSuccess is kept for backward compatibility but is misleading.
+// This is called when client.Start() succeeds, not when a token refresh occurs.
+func LogTokenRefreshSuccess(logger *zap.Logger, duration time.Duration) {
+	LogClientConnectionSuccess(logger, duration)
+}
+
+// Deprecated: Use LogClientConnectionFailure instead.
+// LogTokenRefreshFailure is kept for backward compatibility but is misleading.
+func LogTokenRefreshFailure(logger *zap.Logger, attempt int, err error) {
+	LogClientConnectionFailure(logger, attempt, err)
+}
+
+// LogActualTokenRefreshAttempt logs an actual proactive token refresh attempt.
+// This is called by RefreshManager when it initiates a token refresh operation.
+func LogActualTokenRefreshAttempt(logger *zap.Logger, serverName string, tokenAge time.Duration) {
+	logger.Info("OAuth token refresh attempt",
+		zap.String("server", serverName),
+		zap.Duration("token_age", tokenAge),
+	)
+}
+
+// LogActualTokenRefreshResult logs the result of an actual token refresh operation.
+// This is called by RefreshManager after a refresh attempt completes.
+func LogActualTokenRefreshResult(logger *zap.Logger, serverName string, success bool, duration time.Duration, err error) {
+	if success {
+		logger.Info("OAuth token refresh succeeded",
+			zap.String("server", serverName),
+			zap.Duration("duration", duration),
+		)
+	} else {
+		logger.Warn("OAuth token refresh failed",
+			zap.String("server", serverName),
+			zap.Duration("duration", duration),
+			zap.Error(err),
+		)
+	}
+}
+
 // LogOAuthFlowStart logs the start of an OAuth flow.
 func LogOAuthFlowStart(logger *zap.Logger, serverName string, correlationID string) {
 	logger.Info("Starting OAuth flow",