Skip to content

Commit 70b03e4

Browse files
haimariclaude
andcommitted
fix: Add enhanced retry logic for TerminateInstance rate limiting
- Add RateLimitRetryConfig with 8 attempts, longer delays (up to 120s) - Enhance TerminateInstance to use two-tier retry approach: 1. First attempt with default retry (3 attempts, 30s max) 2. If rate limited, retry with extended backoff (8 attempts, 120s max) - Add explicit "TooManyRequests" and "429" to retryable patterns - Addresses HTTP 429 errors on OCI Compute Service TerminateInstance API 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent 4085371 commit 70b03e4

2 files changed

Lines changed: 36 additions & 1 deletion

File tree

pkg/providers/oci/client.go

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -385,7 +385,8 @@ func (c *Client) TerminateInstance(ctx context.Context, instanceID string) error
385385
logger := log.FromContext(ctx)
386386
logger.Info("terminating OCI instance", "instanceID", instanceID)
387387

388-
return WithRetry(ctx, DefaultRetryConfig(), "terminate-instance", func() error {
388+
// First attempt with default retry config
389+
err := WithRetry(ctx, DefaultRetryConfig(), "terminate-instance", func() error {
389390
request := core.TerminateInstanceRequest{
390391
InstanceId: &instanceID,
391392
PreserveBootVolume: common.Bool(false),
@@ -398,6 +399,28 @@ func (c *Client) TerminateInstance(ctx context.Context, instanceID string) error
398399

399400
return nil
400401
})
402+
403+
// If first attempt failed with rate limiting, retry with more aggressive backoff
404+
if err != nil && IsRateLimitError(err) {
405+
logger.Info("initial terminate attempt hit rate limit, retrying with extended backoff",
406+
"instanceID", instanceID, "error", err)
407+
408+
return WithRetry(ctx, RateLimitRetryConfig(), "terminate-instance-rate-limited", func() error {
409+
request := core.TerminateInstanceRequest{
410+
InstanceId: &instanceID,
411+
PreserveBootVolume: common.Bool(false),
412+
}
413+
414+
_, err := c.computeClient.TerminateInstance(ctx, request)
415+
if err != nil {
416+
return WrapOCIError(err, "instance")
417+
}
418+
419+
return nil
420+
})
421+
}
422+
423+
return err
401424
}
402425

403426
// GetInstance retrieves instance details

pkg/providers/oci/errors.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ func DefaultRetryConfig() RetryConfig {
5555
}
5656
}
5757

58+
// RateLimitRetryConfig returns a retry configuration optimized for rate limiting scenarios
59+
func RateLimitRetryConfig() RetryConfig {
60+
return RetryConfig{
61+
MaxAttempts: 8, // More attempts for rate limiting
62+
InitialDelay: 2 * time.Second, // Start with longer delay
63+
MaxDelay: 120 * time.Second, // Much longer max delay for severe rate limiting
64+
Factor: 2.5, // More aggressive backoff
65+
}
66+
}
67+
5868
// WithRetry executes a function with exponential backoff retry
5969
func WithRetry(ctx context.Context, config RetryConfig, operation string, fn func() error) error {
6070
logger := log.FromContext(ctx)
@@ -136,8 +146,10 @@ func isRetryableError(err error) bool {
136146
"connection refused",
137147
"temporary failure",
138148
"too many requests",
149+
"TooManyRequests",
139150
"rate limit",
140151
"throttled",
152+
"429",
141153
}
142154

143155
for _, pattern := range retryablePatterns {

0 commit comments

Comments
 (0)