From 1abf74659cdeeda51ea3f0b92b0bae36b0cffb4a Mon Sep 17 00:00:00 2001 From: Gautam Raj <156491596+gautam-rl@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:54:50 -0700 Subject: [PATCH 1/2] Add retries on 5xx errors --- cmd/convertor/builder/builder_utils.go | 19 ++++++++++++------- cmd/convertor/main.go | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/convertor/builder/builder_utils.go b/cmd/convertor/builder/builder_utils.go index 3862c106..d49de0cc 100644 --- a/cmd/convertor/builder/builder_utils.go +++ b/cmd/convertor/builder/builder_utils.go @@ -57,14 +57,19 @@ func isRetryableError(err error) bool { // Check for containerd docker error types var dockerErr *docker.Error if errors.As(err, &dockerErr) { - switch dockerErr.Code { - case docker.ErrorCodeTooManyRequests: - return true - case docker.ErrorCodeUnavailable: - return true - default: - return false + // When multiple errors are present, we consider the whole request + // retryable if any of the errors are retryable. + for _, errInfo := range dockerErr.Errors { + switch errInfo.Code { + case docker.ErrorCodeTooManyRequests, // 429 + docker.ErrorCodeUnavailable: // 503 + return true + // Most other 5xx errors are mapped to UNKNOWN. + case docker.ErrorCodeUnknown: + return true + } } + return false } else if err, ok := err.(net.Error); ok && err.Timeout() { return true } diff --git a/cmd/convertor/main.go b/cmd/convertor/main.go index 8eac23a6..223c9a7d 100644 --- a/cmd/convertor/main.go +++ b/cmd/convertor/main.go @@ -396,7 +396,7 @@ func init() { rootCmd.Flags().IntVar(&concurrencyLimit, "concurrency-limit", 4, "the number of manifests that can be built at the same time, used for multi-arch images, 0 means no limit") rootCmd.Flags().BoolVar(&disableSparse, "disable-sparse", false, "disable sparse file for overlaybd") rootCmd.Flags().BoolVar(&referrer, "referrer", false, "push converted manifests with subject, note '--oci' will be enabled automatically if '--referrer' is set, cause the referrer must be in OCI format.") - rootCmd.Flags().IntVar(&retryCount, "retry-count", 5, "number of retries for registry upload operations when encountering 429 rate limiting") + rootCmd.Flags().IntVar(&retryCount, "retry-count", 5, "number of retries for registry upload operations when encountering 429 rate limiting or 5xx errors") // tar import/export rootCmd.Flags().StringVar(&importTar, "import-tar", "", "import image from tar file (OCI layout format)") From 9ee42771e097e6b11dccd7858f788ba7c62c731e Mon Sep 17 00:00:00 2001 From: Gautam Raj <156491596+gautam-rl@users.noreply.github.com> Date: Thu, 11 Sep 2025 15:02:47 -0700 Subject: [PATCH 2/2] fix --- cmd/convertor/builder/builder_utils.go | 41 ++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/cmd/convertor/builder/builder_utils.go b/cmd/convertor/builder/builder_utils.go index d49de0cc..f1d70562 100644 --- a/cmd/convertor/builder/builder_utils.go +++ b/cmd/convertor/builder/builder_utils.go @@ -35,6 +35,7 @@ import ( "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/core/remotes" "github.com/containerd/containerd/v2/core/remotes/docker" + remoteerrors "github.com/containerd/containerd/v2/core/remotes/errors" "github.com/containerd/containerd/v2/pkg/archive/compression" "github.com/containerd/continuity" "github.com/containerd/errdefs" @@ -54,23 +55,33 @@ func isRetryableError(err error) bool { return false } - // Check for containerd docker error types - var dockerErr *docker.Error - if errors.As(err, &dockerErr) { - // When multiple errors are present, we consider the whole request - // retryable if any of the errors are retryable. - for _, errInfo := range dockerErr.Errors { - switch errInfo.Code { - case docker.ErrorCodeTooManyRequests, // 429 - docker.ErrorCodeUnavailable: // 503 - return true - // Most other 5xx errors are mapped to UNKNOWN. - case docker.ErrorCodeUnknown: - return true + // Registry may return multiple errors in one response (docker.Errors) + var regErrs docker.Errors + if errors.As(err, ®Errs) { + for _, e := range regErrs { + switch v := e.(type) { + case docker.ErrorCode: + if v == docker.ErrorCodeTooManyRequests || v == docker.ErrorCodeUnavailable || v == docker.ErrorCodeUnknown { + return true + } + case docker.Error: + if v.Code == docker.ErrorCodeTooManyRequests || v.Code == docker.ErrorCodeUnavailable || v.Code == docker.ErrorCodeUnknown { + return true + } } } - return false - } else if err, ok := err.(net.Error); ok && err.Timeout() { + } + + // Unexpected HTTP status codes (e.g., 429 or 5xx) + var us remoteerrors.ErrUnexpectedStatus + if errors.As(err, &us) { + if us.StatusCode == 429 || (us.StatusCode >= 500 && us.StatusCode <= 599) { + return true + } + } + + // Network timeouts + if ne, ok := err.(net.Error); ok && ne.Timeout() { return true }