Skip to content

Commit d68b694

Browse files
ThomasK33ThomasK33
authored
🤖 fix: back off provisioner reconciliation on coderd 429s (#65)
## Summary This PR hardens `CoderProvisioner` reconciliation against coderd API rate limiting by adding explicit 429 backoff behavior, plus optional rate-limit bypass with fallback in bootstrap client calls. It also includes the existing `flake.nix` dev-tooling update present in this workspace. ## Background Provisioner reconciliation was entering tight error loops after coderd returned HTTP 429 (`Too Many Requests`), which amplified API pressure and prevented stable reconciliation. The operator needed controller-level pacing and safer bootstrap API behavior when bypass headers are unavailable. ## Implementation - Added explicit per-resource jittered exponential backoff for coderd 429s in `CoderProvisionerReconciler`: - base `2s`, cap `2m`, floor `1s`, jitter ratio `0.2` - converts 429 failures into `RequeueAfter` instead of immediate error retries - sets `ProvisionerKeyReady=False` with `Reason=RateLimited` - resets per-resource backoff after non-rate-limited outcomes - Added bootstrap SDK helpers: - `withOptionalRateLimitBypass(...)` - `bypassRateLimitRoundTripper` to inject `X-Coder-Bypass-Ratelimit: true` when requested - automatic retry without bypass when server rejects bypass (`412 Precondition Required`) - exported `IsRateLimitError(err)` helper for controller logic - Applied bypass/fallback flow to: - workspace proxy create/patch operations - provisioner key create/query/delete paths - Added tests: - controller backoff + `RateLimited` condition coverage - workspace proxy bypass fallback coverage - provisioner key ensure/delete bypass fallback coverage - Included existing workspace change in `flake.nix` (`yazi` added to dev shell packages) ## Validation - `make test` - `make build` - `make lint` - `make verify-vendor` ## Risks - **Low-to-medium**: reconcile timing changes for 429 paths only. - Main risk is slower convergence during sustained coderd throttling; this is intentional to prevent hot-loop amplification and API overload. - Scope is limited to bootstrap calls and provisioner reconciliation retry behavior. --- _Generated with `mux` • Model: `openai:gpt-5.3-codex` • Thinking: `xhigh` • Cost: `$1.46`_ <!-- mux-attribution: model=openai:gpt-5.3-codex thinking=xhigh costs=1.46 -->
1 parent a84bece commit d68b694

7 files changed

Lines changed: 443 additions & 35 deletions

File tree

‎flake.nix‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@
5757
govulncheck
5858

5959
docsPython
60+
61+
yazi
6062
];
6163
};
6264
}

‎internal/coderbootstrap/client.go‎

Lines changed: 75 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"context"
66
"errors"
77
"net/http"
8-
"net/url"
98
"time"
109

1110
"golang.org/x/xerrors"
@@ -17,6 +16,65 @@ const (
1716
coderSDKRequestTimeout = 30 * time.Second
1817
)
1918

19+
type bypassRateLimitContextKey struct{}
20+
21+
func withRateLimitBypass(ctx context.Context) context.Context {
22+
if ctx == nil {
23+
return nil
24+
}
25+
return context.WithValue(ctx, bypassRateLimitContextKey{}, true)
26+
}
27+
28+
func shouldBypassRateLimit(ctx context.Context) bool {
29+
if ctx == nil {
30+
return false
31+
}
32+
enabled, _ := ctx.Value(bypassRateLimitContextKey{}).(bool)
33+
return enabled
34+
}
35+
36+
func isRateLimitBypassRejected(err error) bool {
37+
var apiErr *codersdk.Error
38+
return errors.As(err, &apiErr) && apiErr.StatusCode() == http.StatusPreconditionRequired
39+
}
40+
41+
// IsRateLimitError reports whether err (or any wrapped cause) is a codersdk
42+
// API error with HTTP 429 Too Many Requests.
43+
func IsRateLimitError(err error) bool {
44+
var apiErr *codersdk.Error
45+
return errors.As(err, &apiErr) && apiErr.StatusCode() == http.StatusTooManyRequests
46+
}
47+
48+
func withOptionalRateLimitBypass[T any](ctx context.Context, operation func(context.Context) (T, error)) (T, error) {
49+
result, err := operation(withRateLimitBypass(ctx))
50+
if err == nil {
51+
return result, nil
52+
}
53+
if !isRateLimitBypassRejected(err) {
54+
return result, err
55+
}
56+
return operation(ctx)
57+
}
58+
59+
type bypassRateLimitRoundTripper struct {
60+
base http.RoundTripper
61+
}
62+
63+
func (rt bypassRateLimitRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
64+
if req == nil {
65+
return nil, xerrors.New("assertion failed: request must not be nil")
66+
}
67+
if !shouldBypassRateLimit(req.Context()) {
68+
return rt.base.RoundTrip(req)
69+
}
70+
71+
cloned := req.Clone(req.Context())
72+
cloned.Header = req.Header.Clone()
73+
cloned.Header.Set(codersdk.BypassRatelimitHeader, "true")
74+
75+
return rt.base.RoundTrip(cloned)
76+
}
77+
2078
// RegisterWorkspaceProxyRequest describes how to register a workspace proxy in Coder.
2179
type RegisterWorkspaceProxyRequest struct {
2280
CoderURL string
@@ -60,24 +118,10 @@ func (c *SDKClient) EnsureWorkspaceProxy(ctx context.Context, req RegisterWorksp
60118
return RegisterWorkspaceProxyResponse{}, xerrors.New("proxy name is required")
61119
}
62120

63-
coderURL, err := url.Parse(req.CoderURL)
121+
client, err := newAuthenticatedClient(req.CoderURL, req.SessionToken)
64122
if err != nil {
65-
return RegisterWorkspaceProxyResponse{}, xerrors.Errorf("parse coder URL: %w", err)
66-
}
67-
68-
client := codersdk.New(coderURL)
69-
client.SetSessionToken(req.SessionToken)
70-
if client.HTTPClient == nil {
71-
client.HTTPClient = &http.Client{}
72-
}
73-
defaultTransport, ok := http.DefaultTransport.(*http.Transport)
74-
if !ok {
75-
return RegisterWorkspaceProxyResponse{}, xerrors.New("assertion failed: http.DefaultTransport is not *http.Transport")
123+
return RegisterWorkspaceProxyResponse{}, err
76124
}
77-
// Use a dedicated transport to avoid sharing http.DefaultTransport's
78-
// connection pool across parallel test servers.
79-
client.HTTPClient.Transport = defaultTransport.Clone()
80-
client.HTTPClient.Timeout = coderSDKRequestTimeout
81125

82126
existing, err := client.WorkspaceProxyByName(ctx, req.ProxyName)
83127
if err != nil {
@@ -86,10 +130,12 @@ func (c *SDKClient) EnsureWorkspaceProxy(ctx context.Context, req RegisterWorksp
86130
return RegisterWorkspaceProxyResponse{}, xerrors.Errorf("query workspace proxy %q: %w", req.ProxyName, err)
87131
}
88132

89-
created, createErr := client.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
90-
Name: req.ProxyName,
91-
DisplayName: req.DisplayName,
92-
Icon: req.Icon,
133+
created, createErr := withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (codersdk.UpdateWorkspaceProxyResponse, error) {
134+
return client.CreateWorkspaceProxy(requestCtx, codersdk.CreateWorkspaceProxyRequest{
135+
Name: req.ProxyName,
136+
DisplayName: req.DisplayName,
137+
Icon: req.Icon,
138+
})
93139
})
94140
if createErr != nil {
95141
return RegisterWorkspaceProxyResponse{}, xerrors.Errorf("create workspace proxy %q: %w", req.ProxyName, createErr)
@@ -109,12 +155,14 @@ func (c *SDKClient) EnsureWorkspaceProxy(ctx context.Context, req RegisterWorksp
109155
icon = existing.IconURL
110156
}
111157

112-
updated, err := client.PatchWorkspaceProxy(ctx, codersdk.PatchWorkspaceProxy{
113-
ID: existing.ID,
114-
Name: existing.Name,
115-
DisplayName: displayName,
116-
Icon: icon,
117-
RegenerateToken: true,
158+
updated, err := withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (codersdk.UpdateWorkspaceProxyResponse, error) {
159+
return client.PatchWorkspaceProxy(requestCtx, codersdk.PatchWorkspaceProxy{
160+
ID: existing.ID,
161+
Name: existing.Name,
162+
DisplayName: displayName,
163+
Icon: icon,
164+
RegenerateToken: true,
165+
})
118166
})
119167
if err != nil {
120168
return RegisterWorkspaceProxyResponse{}, xerrors.Errorf("update workspace proxy %q: %w", req.ProxyName, err)

‎internal/coderbootstrap/client_test.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"testing"
99
"time"
1010

11+
"github.com/coder/coder/v2/codersdk"
1112
"github.com/google/uuid"
1213
"github.com/stretchr/testify/require"
1314

@@ -200,6 +201,72 @@ func TestEnsureWorkspaceProxyUpdatesExistingProxy(t *testing.T) {
200201
require.Equal(t, "token-updated", result.ProxyToken)
201202
}
202203

204+
func TestEnsureWorkspaceProxyCreateFallsBackWhenRateLimitBypassRejected(t *testing.T) {
205+
t.Parallel()
206+
207+
const proxyName = "proxy-fallback"
208+
now := time.Now().UTC().Format(time.RFC3339)
209+
proxyID := uuid.NewString()
210+
createCalls := 0
211+
bypassRejectedCalls := 0
212+
213+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
214+
switch {
215+
case r.Method == http.MethodGet && r.URL.Path == "/api/v2/workspaceproxies/"+proxyName:
216+
w.WriteHeader(http.StatusNotFound)
217+
_ = json.NewEncoder(w).Encode(map[string]string{"message": "not found"})
218+
return
219+
case r.Method == http.MethodPost && r.URL.Path == "/api/v2/workspaceproxies":
220+
createCalls++
221+
if r.Header.Get(codersdk.BypassRatelimitHeader) == "true" {
222+
bypassRejectedCalls++
223+
w.WriteHeader(http.StatusPreconditionRequired)
224+
_ = json.NewEncoder(w).Encode(map[string]string{"message": "bypass is not allowed"})
225+
return
226+
}
227+
228+
response := proxyResponse{}
229+
response.Proxy.ID = proxyID
230+
response.Proxy.Name = proxyName
231+
response.Proxy.DisplayName = "Proxy Fallback"
232+
response.Proxy.IconURL = "/emojis/1f5fa.png"
233+
response.Proxy.Healthy = true
234+
response.Proxy.PathAppURL = "https://proxy-fallback.example.com"
235+
response.Proxy.WildcardHostname = "*.proxy-fallback.example.com"
236+
response.Proxy.Status.Status = "unregistered"
237+
response.Proxy.Status.CheckedAt = now
238+
response.Proxy.CreatedAt = now
239+
response.Proxy.UpdatedAt = now
240+
response.Proxy.Version = "2.0.0"
241+
response.ProxyToken = "token-created"
242+
243+
w.WriteHeader(http.StatusCreated)
244+
err := json.NewEncoder(w).Encode(response)
245+
require.NoError(t, err)
246+
return
247+
default:
248+
w.WriteHeader(http.StatusNotFound)
249+
_ = json.NewEncoder(w).Encode(map[string]string{"message": "unexpected route"})
250+
return
251+
}
252+
}))
253+
defer server.Close()
254+
255+
client := coderbootstrap.NewSDKClient()
256+
result, err := client.EnsureWorkspaceProxy(context.Background(), coderbootstrap.RegisterWorkspaceProxyRequest{
257+
CoderURL: server.URL,
258+
SessionToken: "session-token",
259+
ProxyName: proxyName,
260+
DisplayName: "Proxy Fallback",
261+
Icon: "/emojis/1f5fa.png",
262+
})
263+
require.NoError(t, err)
264+
require.Equal(t, proxyName, result.ProxyName)
265+
require.Equal(t, "token-created", result.ProxyToken)
266+
require.Equal(t, 2, createCalls)
267+
require.Equal(t, 1, bypassRejectedCalls)
268+
}
269+
203270
func TestEnsureWorkspaceProxyValidatesInputs(t *testing.T) {
204271
t.Parallel()
205272

‎internal/coderbootstrap/provisionerkeys.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,11 @@ func (c *SDKClient) EnsureProvisionerKey(ctx context.Context, req EnsureProvisio
6767
}, nil
6868
}
6969

70-
created, err := client.CreateProvisionerKey(ctx, organization.ID, codersdk.CreateProvisionerKeyRequest{
71-
Name: req.KeyName,
72-
Tags: req.Tags,
70+
created, err := withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (codersdk.CreateProvisionerKeyResponse, error) {
71+
return client.CreateProvisionerKey(requestCtx, organization.ID, codersdk.CreateProvisionerKeyRequest{
72+
Name: req.KeyName,
73+
Tags: req.Tags,
74+
})
7375
})
7476
if err != nil {
7577
return EnsureProvisionerKeyResponse{}, xerrors.Errorf("create provisioner key %q: %w", req.KeyName, err)
@@ -78,7 +80,9 @@ func (c *SDKClient) EnsureProvisionerKey(ctx context.Context, req EnsureProvisio
7880
return EnsureProvisionerKeyResponse{}, xerrors.Errorf("assertion failed: created provisioner key %q returned an empty key", req.KeyName)
7981
}
8082

81-
createdMetadata, err := findOrganizationProvisionerKey(ctx, client, organization.ID, req.KeyName)
83+
createdMetadata, err := withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (*codersdk.ProvisionerKey, error) {
84+
return findOrganizationProvisionerKey(requestCtx, client, organization.ID, req.KeyName)
85+
})
8286
if err != nil {
8387
return EnsureProvisionerKeyResponse{}, xerrors.Errorf("query created provisioner key %q: %w", req.KeyName, err)
8488
}
@@ -119,7 +123,9 @@ func (c *SDKClient) DeleteProvisionerKey(ctx context.Context, coderURL, sessionT
119123
return err
120124
}
121125

122-
err = client.DeleteProvisionerKey(ctx, organization.ID, keyName)
126+
_, err = withOptionalRateLimitBypass(ctx, func(requestCtx context.Context) (struct{}, error) {
127+
return struct{}{}, client.DeleteProvisionerKey(requestCtx, organization.ID, keyName)
128+
})
123129
if err == nil {
124130
return nil
125131
}
@@ -166,7 +172,7 @@ func newAuthenticatedClient(coderURL, sessionToken string) (*codersdk.Client, er
166172
}
167173
// Use a dedicated transport to avoid sharing http.DefaultTransport's
168174
// connection pool across parallel test servers.
169-
client.HTTPClient.Transport = defaultTransport.Clone()
175+
client.HTTPClient.Transport = bypassRateLimitRoundTripper{base: defaultTransport.Clone()}
170176
client.HTTPClient.Timeout = coderSDKRequestTimeout
171177

172178
return client, nil

0 commit comments

Comments
 (0)