Skip to content

Commit 8e6a547

Browse files
gautam-rlgautam-rl
committed
Retry on docker registry upload failures
1 parent 3a38a2b commit 8e6a547

6 files changed

Lines changed: 122 additions & 37 deletions

File tree

‎cmd/convertor/builder/builder.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ type BuilderOptions struct {
7575
// Push manifests with subject
7676
Referrer bool
7777

78+
// Number of retries for registry upload operations when encountering 429 rate limiting
79+
RetryCount int
80+
7881
// CustomResolver allows using a custom resolver instead of the default docker resolver
7982
// Used for tar import/export functionality
8083
CustomResolver remotes.Resolver
@@ -223,7 +226,7 @@ func (b *graphBuilder) process(ctx context.Context, src v1.Descriptor, tag bool)
223226
} else {
224227
pusher = b.pusher
225228
}
226-
if err := uploadBytes(ctx, pusher, expected, indexBytes); err != nil {
229+
if err := uploadBytesWithRetry(ctx, pusher, expected, indexBytes, b.RetryCount); err != nil {
227230
return v1.Descriptor{}, fmt.Errorf("failed to upload index: %w", err)
228231
}
229232
log.G(ctx).Infof("index uploaded, %s", expected.Digest)
@@ -296,6 +299,7 @@ func (b *graphBuilder) buildOne(ctx context.Context, src v1.Descriptor, tag bool
296299
engineBase.reserve = b.Reserve
297300
engineBase.noUpload = b.NoUpload
298301
engineBase.dumpManifest = b.DumpManifest
302+
engineBase.retryCount = b.RetryCount
299303
if _, ok := b.Resolver.(*FileBasedResolver); ok {
300304
engineBase.tarExport = true
301305
}

‎cmd/convertor/builder/builder_engine.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ type builderEngineBase struct {
117117
dumpManifest bool
118118
referrer bool
119119
tarExport bool
120+
retryCount int
120121
}
121122

122123
func (e *builderEngineBase) isGzipLayer(ctx context.Context, idx int) (bool, error) {
@@ -183,7 +184,7 @@ func (e *builderEngineBase) uploadManifestAndConfig(ctx context.Context) (specs.
183184
Size: (int64)(len(cbuf)),
184185
}
185186
if shouldUploadBlob {
186-
if err = uploadBytes(ctx, e.pusher, e.manifest.Config, cbuf); err != nil {
187+
if err = uploadBytesWithRetry(ctx, e.pusher, e.manifest.Config, cbuf, e.retryCount); err != nil {
187188
return specs.Descriptor{}, errors.Wrapf(err, "failed to upload config")
188189
}
189190
log.G(ctx).Infof("config uploaded")
@@ -207,7 +208,7 @@ func (e *builderEngineBase) uploadManifestAndConfig(ctx context.Context) (specs.
207208
Size: (int64)(len(cbuf)),
208209
}
209210
if shouldUploadBlob {
210-
if err = uploadBytes(ctx, e.pusher, manifestDesc, cbuf); err != nil {
211+
if err = uploadBytesWithRetry(ctx, e.pusher, manifestDesc, cbuf, e.retryCount); err != nil {
211212
return specs.Descriptor{}, errors.Wrapf(err, "failed to upload manifest")
212213
}
213214
e.outputDesc = manifestDesc

‎cmd/convertor/builder/builder_utils.go‎

Lines changed: 103 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,15 @@ import (
2424
"encoding/json"
2525
"fmt"
2626
"io"
27+
"math"
2728
"os"
2829
"path"
30+
"time"
2931

3032
"github.com/containerd/containerd/v2/core/content"
3133
"github.com/containerd/containerd/v2/core/images"
3234
"github.com/containerd/containerd/v2/core/remotes"
35+
"github.com/containerd/containerd/v2/core/remotes/docker"
3336
"github.com/containerd/containerd/v2/pkg/archive/compression"
3437
"github.com/containerd/continuity"
3538
"github.com/containerd/errdefs"
@@ -43,6 +46,63 @@ import (
4346
t "github.com/containerd/accelerated-container-image/pkg/types"
4447
)
4548

49+
// isRetryableError checks if the error is retryable (429 or 5xx errors)
50+
func isRetryableError(err error) bool {
51+
if err == nil {
52+
return false
53+
}
54+
55+
// Check for containerd docker error types
56+
var dockerErr *docker.Error
57+
if errors.As(err, &dockerErr) {
58+
switch dockerErr.Code {
59+
case docker.ErrorCodeTooManyRequests:
60+
return true
61+
case docker.ErrorCodeUnavailable:
62+
return true
63+
default:
64+
return false
65+
}
66+
}
67+
68+
return false
69+
}
70+
71+
// retryWithBackoff executes a function with exponential backoff on retryable errors
72+
func retryWithBackoff(ctx context.Context, maxRetries int, operation func() error) error {
73+
var lastErr error
74+
75+
for attempt := 0; attempt <= maxRetries; attempt++ {
76+
lastErr = operation()
77+
78+
if lastErr == nil {
79+
return nil
80+
}
81+
82+
if !isRetryableError(lastErr) {
83+
return lastErr
84+
}
85+
86+
if attempt == maxRetries {
87+
logrus.Warnf("max retries (%d) reached for retryable error: %v", maxRetries, lastErr)
88+
return lastErr
89+
}
90+
91+
// Exponential backoff: base delay of 1s, max 30s
92+
backoffDelay := time.Duration(math.Min(float64(time.Second)*math.Pow(2, float64(attempt)), float64(30*time.Second)))
93+
logrus.Infof("received retryable error, retrying in %v (attempt %d/%d): %v", backoffDelay, attempt+1, maxRetries, lastErr)
94+
95+
select {
96+
case <-ctx.Done():
97+
return ctx.Err()
98+
case <-time.After(backoffDelay):
99+
continue
100+
}
101+
}
102+
103+
return lastErr
104+
}
105+
46106
func fetch(ctx context.Context, fetcher remotes.Fetcher, desc specs.Descriptor, target any) error {
47107
rc, err := fetcher.Fetch(ctx, desc)
48108
if err != nil {
@@ -200,41 +260,57 @@ func getFileDesc(filepath string, decompress bool) (specs.Descriptor, error) {
200260
}
201261

202262
func uploadBlob(ctx context.Context, pusher remotes.Pusher, path string, desc specs.Descriptor) error {
203-
cw, err := pusher.Push(ctx, desc)
204-
if err != nil {
205-
if errdefs.IsAlreadyExists(err) {
206-
logrus.Infof("layer %s exists", desc.Digest.String())
207-
return nil
263+
return uploadBlobWithRetry(ctx, pusher, path, desc, 0)
264+
}
265+
266+
func uploadBlobWithRetry(ctx context.Context, pusher remotes.Pusher, path string, desc specs.Descriptor, retryCount int) error {
267+
return retryWithBackoff(ctx, retryCount, func() error {
268+
cw, err := pusher.Push(ctx, desc)
269+
if err != nil {
270+
if errdefs.IsAlreadyExists(err) {
271+
logrus.Infof("layer %s exists", desc.Digest.String())
272+
return nil
273+
}
274+
return err
208275
}
209-
return err
210-
}
211276

212-
defer cw.Close()
213-
fobd, err := os.Open(path)
214-
if err != nil {
215-
return err
216-
}
217-
defer fobd.Close()
218-
if err = content.Copy(ctx, cw, fobd, desc.Size, desc.Digest); err != nil {
219-
return err
220-
}
221-
return nil
277+
defer cw.Close()
278+
fobd, err := os.Open(path)
279+
if err != nil {
280+
return err
281+
}
282+
defer fobd.Close()
283+
if err = content.Copy(ctx, cw, fobd, desc.Size, desc.Digest); err != nil {
284+
return err
285+
}
286+
return nil
287+
})
222288
}
223289

224290
func uploadBytes(ctx context.Context, pusher remotes.Pusher, desc specs.Descriptor, data []byte) error {
225-
cw, err := pusher.Push(ctx, desc)
226-
if err != nil {
227-
if errdefs.IsAlreadyExists(err) {
228-
logrus.Infof("content %s exists", desc.Digest.String())
229-
return nil
291+
return uploadBytesWithRetry(ctx, pusher, desc, data, 0)
292+
}
293+
294+
func uploadBytesWithRetry(ctx context.Context, pusher remotes.Pusher, desc specs.Descriptor, data []byte, retryCount int) error {
295+
return retryWithBackoff(ctx, retryCount, func() error {
296+
cw, err := pusher.Push(ctx, desc)
297+
if err != nil {
298+
if errdefs.IsAlreadyExists(err) {
299+
logrus.Infof("content %s exists", desc.Digest.String())
300+
return nil
301+
}
302+
return err
230303
}
231-
return err
232-
}
233-
defer cw.Close()
234-
return content.Copy(ctx, cw, bytes.NewReader(data), desc.Size, desc.Digest)
304+
defer cw.Close()
305+
return content.Copy(ctx, cw, bytes.NewReader(data), desc.Size, desc.Digest)
306+
})
235307
}
236308

237309
func tagPreviouslyConvertedManifest(ctx context.Context, pusher remotes.Pusher, fetcher remotes.Fetcher, desc specs.Descriptor) error {
310+
return tagPreviouslyConvertedManifestWithRetry(ctx, pusher, fetcher, desc, 0)
311+
}
312+
313+
func tagPreviouslyConvertedManifestWithRetry(ctx context.Context, pusher remotes.Pusher, fetcher remotes.Fetcher, desc specs.Descriptor, retryCount int) error {
238314
manifest := specs.Manifest{}
239315
if err := fetch(ctx, fetcher, desc, &manifest); err != nil {
240316
return fmt.Errorf("failed to fetch converted manifest: %w", err)
@@ -243,7 +319,7 @@ func tagPreviouslyConvertedManifest(ctx context.Context, pusher remotes.Pusher,
243319
if err != nil {
244320
return err
245321
}
246-
if err := uploadBytes(ctx, pusher, desc, cbuf); err != nil {
322+
if err := uploadBytesWithRetry(ctx, pusher, desc, cbuf, retryCount); err != nil {
247323
return fmt.Errorf("failed to tag converted manifest: %w", err)
248324
}
249325
return nil

‎cmd/convertor/builder/overlaybd_builder.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ func (e *overlaybdBuilderEngine) UploadLayer(ctx context.Context, idx int) error
192192
}
193193
shouldUploadBlob := !e.noUpload || e.tarExport
194194
if shouldUploadBlob {
195-
if err := uploadBlob(ctx, e.pusher, path.Join(layerDir, commitFile), desc); err != nil {
195+
if err := uploadBlobWithRetry(ctx, e.pusher, path.Join(layerDir, commitFile), desc, e.retryCount); err != nil {
196196
return errors.Wrapf(err, "failed to upload layer %d", idx)
197197
}
198198
}
@@ -354,7 +354,7 @@ func (e *overlaybdBuilderEngine) CheckForConvertedManifest(ctx context.Context)
354354

355355
// If a converted manifest has been found we still need to tag it to match the expected output tag.
356356
func (e *overlaybdBuilderEngine) TagPreviouslyConvertedManifest(ctx context.Context, desc specs.Descriptor) error {
357-
return tagPreviouslyConvertedManifest(ctx, e.pusher, e.fetcher, desc)
357+
return tagPreviouslyConvertedManifestWithRetry(ctx, e.pusher, e.fetcher, desc, e.retryCount)
358358
}
359359

360360
// mountImage is responsible for mounting a specific manifest from a source repository, this includes
@@ -391,7 +391,7 @@ func (e *overlaybdBuilderEngine) mountImage(ctx context.Context, manifest specs.
391391
if err != nil {
392392
return err
393393
}
394-
return uploadBytes(ctx, e.pusher, desc, cbuf)
394+
return uploadBytesWithRetry(ctx, e.pusher, desc, cbuf, e.retryCount)
395395
}
396396

397397
func (e *overlaybdBuilderEngine) StoreConvertedManifestDetails(ctx context.Context) error {
@@ -483,7 +483,7 @@ func (e *overlaybdBuilderEngine) uploadBaseLayer(ctx context.Context) (specs.Des
483483
}
484484
shouldUploadBlob := !e.noUpload || e.tarExport
485485
if shouldUploadBlob {
486-
if err = uploadBlob(ctx, e.pusher, tarFile, baseDesc); err != nil {
486+
if err = uploadBlobWithRetry(ctx, e.pusher, tarFile, baseDesc, e.retryCount); err != nil {
487487
return specs.Descriptor{}, errors.Wrapf(err, "failed to upload baselayer")
488488
}
489489
logrus.Infof("baselayer uploaded")

‎cmd/convertor/builder/turboOCI_builder.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ func (e *turboOCIBuilderEngine) UploadLayer(ctx context.Context, idx int) error
168168
}
169169
}
170170
desc.Annotations[label.TurboOCIMediaType] = targetMediaType
171-
if err := uploadBlob(ctx, e.pusher, path.Join(layerDir, tociLayerTar), desc); err != nil {
171+
if err := uploadBlobWithRetry(ctx, e.pusher, path.Join(layerDir, tociLayerTar), desc, e.retryCount); err != nil {
172172
return errors.Wrapf(err, "failed to upload layer %d", idx)
173173
}
174174
e.tociLayers[idx] = desc
@@ -196,7 +196,7 @@ func (e *turboOCIBuilderEngine) UploadImage(ctx context.Context) (specs.Descript
196196
},
197197
}
198198
if !e.mkfs {
199-
if err := uploadBlob(ctx, e.pusher, overlaybdBaseLayer, baseDesc); err != nil {
199+
if err := uploadBlobWithRetry(ctx, e.pusher, overlaybdBaseLayer, baseDesc, e.retryCount); err != nil {
200200
return specs.Descriptor{}, errors.Wrapf(err, "failed to upload baselayer %q", overlaybdBaseLayer)
201201
}
202202
e.manifest.Layers = append([]specs.Descriptor{baseDesc}, e.manifest.Layers...)
@@ -215,7 +215,7 @@ func (e *turboOCIBuilderEngine) UploadImage(ctx context.Context) (specs.Descript
215215

216216
// If a converted manifest has been found we still need to tag it to match the expected output tag.
217217
func (e *turboOCIBuilderEngine) TagPreviouslyConvertedManifest(ctx context.Context, desc specs.Descriptor) error {
218-
return tagPreviouslyConvertedManifest(ctx, e.pusher, e.fetcher, desc)
218+
return tagPreviouslyConvertedManifestWithRetry(ctx, e.pusher, e.fetcher, desc, e.retryCount)
219219
}
220220

221221
// Layer deduplication in FastOCI is not currently supported due to conversion not

‎cmd/convertor/main.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ var (
5959
concurrencyLimit int
6060
disableSparse bool
6161
referrer bool
62+
retryCount int
6263

6364
// tar import/export
6465
importTar string
@@ -279,6 +280,7 @@ Version: ` + commitID,
279280
ConcurrencyLimit: concurrencyLimit,
280281
DisableSparse: disableSparse,
281282
Referrer: referrer,
283+
RetryCount: retryCount,
282284
}
283285
} else {
284286
// Normal registry mode
@@ -307,6 +309,7 @@ Version: ` + commitID,
307309
ConcurrencyLimit: concurrencyLimit,
308310
DisableSparse: disableSparse,
309311
Referrer: referrer,
312+
RetryCount: retryCount,
310313
}
311314
}
312315
if overlaybd != "" {
@@ -393,6 +396,7 @@ func init() {
393396
rootCmd.Flags().IntVar(&concurrencyLimit, "concurrency-limit", 4, "the number of manifests that can be built at the same time, used for multi-arch images, 0 means no limit")
394397
rootCmd.Flags().BoolVar(&disableSparse, "disable-sparse", false, "disable sparse file for overlaybd")
395398
rootCmd.Flags().BoolVar(&referrer, "referrer", false, "push converted manifests with subject, note '--oci' will be enabled automatically if '--referrer' is set, cause the referrer must be in OCI format.")
399+
rootCmd.Flags().IntVar(&retryCount, "retry-count", 5, "number of retries for registry upload operations when encountering 429 rate limiting")
396400

397401
// tar import/export
398402
rootCmd.Flags().StringVar(&importTar, "import-tar", "", "import image from tar file (OCI layout format)")

0 commit comments

Comments
 (0)