|
| 1 | +package auth |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "errors" |
| 6 | + "net" |
| 7 | + "net/http" |
| 8 | + "slices" |
| 9 | + "strconv" |
| 10 | + "syscall" |
| 11 | + "time" |
| 12 | + |
| 13 | + "github.com/databricks/databricks-sdk-go/experimental/api" |
| 14 | + "golang.org/x/oauth2" |
| 15 | +) |
| 16 | + |
| 17 | +// retryingTokenSource wraps a TokenSource with retry logic for transient |
| 18 | +// failures during token acquisition. Each retry calls the underlying Token() |
| 19 | +// method, which creates a fresh HTTP request. |
| 20 | +type retryingTokenSource struct { |
| 21 | + inner TokenSource |
| 22 | + opts []api.Option |
| 23 | +} |
| 24 | + |
| 25 | +// NewRetryingTokenSource wraps inner with retry logic for transient failures. |
| 26 | +// The provided options are applied after the default options, allowing callers |
| 27 | +// to override the default timeout and retry behavior. |
| 28 | +func NewRetryingTokenSource(inner TokenSource, opts ...api.Option) TokenSource { |
| 29 | + defaultOptions := []api.Option{ |
| 30 | + api.WithTimeout(1 * time.Minute), |
| 31 | + api.WithRetrier(func() api.Retrier { return &httpRetrier{} }), |
| 32 | + } |
| 33 | + return &retryingTokenSource{ |
| 34 | + inner: inner, |
| 35 | + opts: append(defaultOptions, opts...), |
| 36 | + } |
| 37 | +} |
| 38 | + |
| 39 | +// Token returns a token from the underlying source, retrying on transient |
| 40 | +// errors. |
| 41 | +func (r *retryingTokenSource) Token(ctx context.Context) (*oauth2.Token, error) { |
| 42 | + return api.ExecuteWithResult(ctx, r.inner.Token, r.opts...) |
| 43 | +} |
| 44 | + |
| 45 | +// httpRetrier classifies errors from HTTP-based token endpoints and determines |
| 46 | +// whether they should be retried and with what delay. |
| 47 | +// |
| 48 | +// An error is retriable if it carries an HTTP status code in retriableCodes |
| 49 | +// (429, 502, 503, 504) or is a transient network error (ECONNRESET, |
| 50 | +// ECONNREFUSED, timeout). All other errors, including 4xx client errors, are |
| 51 | +// not retriable regardless of headers. |
| 52 | +// |
| 53 | +// For retriable HTTP errors, the Retry-After header is used as a hint: the |
| 54 | +// delay is the maximum of the exponential backoff and the Retry-After value, |
| 55 | +// so we never retry sooner than the server asked. No upper cap is applied, |
| 56 | +// the context timeout already bounds total retry duration. |
| 57 | +// |
| 58 | +// TODO: this retrier encodes logic (retriable status codes, Retry-After |
| 59 | +// parsing, network error classification) that is also needed by the main HTTP |
| 60 | +// retry loop in httpclient/. Consider moving it to a shared location so both |
| 61 | +// token acquisition and regular API calls use the same retry classification. |
| 62 | +type httpRetrier struct { |
| 63 | + backoff api.BackoffPolicy |
| 64 | +} |
| 65 | + |
| 66 | +// IsRetriable reports whether err is a transient failure that should be |
| 67 | +// retried, and if so, how long the caller should wait before retrying. |
| 68 | +func (r *httpRetrier) IsRetriable(err error) (time.Duration, bool) { |
| 69 | + if code, header := httpResponse(err); code != 0 { |
| 70 | + if !slices.Contains(retriableCodes, code) { |
| 71 | + return 0, false |
| 72 | + } |
| 73 | + delay := r.backoff.Delay() |
| 74 | + if d, ok := parseRetryAfter(header); ok && d > delay { |
| 75 | + delay = d // context timeout already bounds total retry duration |
| 76 | + } |
| 77 | + return delay, true |
| 78 | + } |
| 79 | + if isTransientNetworkError(err) { |
| 80 | + return r.backoff.Delay(), true |
| 81 | + } |
| 82 | + return 0, false |
| 83 | +} |
| 84 | + |
| 85 | +// parseRetryAfter parses a Retry-After header value as either a delay in |
| 86 | +// seconds or an HTTP-date. |
| 87 | +func parseRetryAfter(header http.Header) (time.Duration, bool) { |
| 88 | + if header == nil { |
| 89 | + return 0, false |
| 90 | + } |
| 91 | + v := header.Get("Retry-After") |
| 92 | + if v == "" { |
| 93 | + return 0, false |
| 94 | + } |
| 95 | + if seconds, parseErr := strconv.Atoi(v); parseErr == nil && seconds >= 0 { |
| 96 | + return time.Duration(seconds) * time.Second, true |
| 97 | + } |
| 98 | + if t, parseErr := http.ParseTime(v); parseErr == nil { |
| 99 | + if d := time.Until(t); d > 0 { |
| 100 | + return d, true |
| 101 | + } |
| 102 | + } |
| 103 | + return 0, false |
| 104 | +} |
| 105 | + |
| 106 | +// httpResponseError is implemented by errors that carry HTTP response |
| 107 | +// metadata. This interface avoids importing httpclient (which would create a |
| 108 | +// cycle) while still allowing classification of httpclient.HttpError by status |
| 109 | +// code and headers. |
| 110 | +// |
| 111 | +// TODO: This is meant to be temporary and should move this to a shared location |
| 112 | +// so both token acquisition and regular API calls use the same classification. |
| 113 | +type httpResponseError interface { |
| 114 | + HTTPStatusCode() int |
| 115 | + Header() http.Header |
| 116 | +} |
| 117 | + |
| 118 | +var retriableCodes = []int{ |
| 119 | + http.StatusTooManyRequests, // 429 |
| 120 | + http.StatusBadGateway, // 502 |
| 121 | + http.StatusServiceUnavailable, // 503 |
| 122 | + http.StatusGatewayTimeout, // 504 |
| 123 | +} |
| 124 | + |
| 125 | +// httpResponse extracts HTTP response metadata from an error, if available. |
| 126 | +func httpResponse(err error) (statusCode int, header http.Header) { |
| 127 | + var retrieveErr *oauth2.RetrieveError |
| 128 | + if errors.As(err, &retrieveErr) && retrieveErr.Response != nil { |
| 129 | + return retrieveErr.Response.StatusCode, retrieveErr.Response.Header |
| 130 | + } |
| 131 | + var re httpResponseError |
| 132 | + if errors.As(err, &re) { |
| 133 | + return re.HTTPStatusCode(), re.Header() |
| 134 | + } |
| 135 | + return 0, nil |
| 136 | +} |
| 137 | + |
| 138 | +// isTransientNetworkError reports whether err represents a transient network |
| 139 | +// condition that is likely to resolve on retry. |
| 140 | +func isTransientNetworkError(err error) bool { |
| 141 | + if errors.Is(err, syscall.ECONNRESET) { |
| 142 | + return true |
| 143 | + } |
| 144 | + if errors.Is(err, syscall.ECONNREFUSED) { |
| 145 | + return true |
| 146 | + } |
| 147 | + var netErr net.Error |
| 148 | + if errors.As(err, &netErr) && netErr.Timeout() { |
| 149 | + return true |
| 150 | + } |
| 151 | + return false |
| 152 | +} |
0 commit comments