Retry in case of 409 conflict error

clm/cmd/apply.go

@@ -7,6 +7,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -17,7 +18,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	apitypes "k8s.io/apimachinery/pkg/types"
-	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/sap/component-operator-runtime/clm/internal/backoff"
 	"github.com/sap/component-operator-runtime/clm/internal/manifests"
@@ -108,7 +108,7 @@ func newApplyCmd() *cobra.Command {
 					release.State = component.StateError
 				}
 				if updateErr := releaseClient.Update(context.TODO(), release); updateErr != nil {
-					err = utilerrors.NewAggregate([]error{err, updateErr})
+					err = errors.Join(err, updateErr)
 				}
 			}()
 

clm/cmd/delete.go

@@ -7,12 +7,12 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"time"
 
 	"github.com/spf13/cobra"
-	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/sap/component-operator-runtime/clm/internal/backoff"
 	"github.com/sap/component-operator-runtime/clm/internal/release"
@@ -87,7 +87,7 @@ func newDeleteCmd() *cobra.Command {
 					release.State = component.StateError
 				}
 				if updateErr := releaseClient.Update(context.TODO(), release); updateErr != nil {
-					err = utilerrors.NewAggregate([]error{err, updateErr})
+					err = errors.Join(err, updateErr)
 				}
 			}()
 

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/go-git/go-git v4.7.0+incompatible
 	github.com/go-logr/logr v1.4.3
 	github.com/gobwas/glob v0.2.3
-	github.com/hashicorp/go-multierror v1.1.1
 	github.com/iancoleman/strcase v0.3.0
 	github.com/onsi/ginkgo/v2 v2.27.2
 	github.com/onsi/gomega v1.38.2
@@ -60,7 +59,6 @@ require (
 	github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect

go.sum

@@ -102,11 +102,6 @@ github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJr
 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
-github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=

internal/clientfactory/factory.go

@@ -9,7 +9,6 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"reflect"
 	"sync"
 	"time"
@@ -121,12 +120,6 @@ func (f *ClientFactory) Get(kubeConfig []byte, impersonationUser string, imperso
 		return clnt, nil
 	}
 
-	config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
-		return roundTripperFunc(func(r *http.Request) (*http.Response, error) {
-			metrics.Requests.WithLabelValues(f.controllerName, r.Method).Inc()
-			return rt.RoundTrip(r)
-		})
-	})
 	clnt, err := NewClientFor(config, f.scheme, f.name)
 	if err != nil {
 		return nil, err
@@ -149,11 +142,3 @@ func sha256sum(data any) string {
 	sha256sum := sha256.Sum256(dataAsJson)
 	return string(sha256sum[:])
 }
-
-type roundTripperFunc func(r *http.Request) (*http.Response, error)
-
-var _ http.RoundTripper = roundTripperFunc(nil)
-
-func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
-	return f(r)
-}

internal/helm/hooks.go

@@ -10,7 +10,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pkg/errors"
+	legacyerrors "github.com/pkg/errors"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -44,7 +44,7 @@ func ParseHookMetadata(object client.Object) (*HookMetadata, error) {
 	if value, ok := annotations[annotationKeyHookWeight]; ok {
 		weight, err := strconv.Atoi(value)
 		if err != nil {
-			return nil, errors.Wrapf(err, "invalid hook weight: %s", value)
+			return nil, legacyerrors.Wrapf(err, "invalid hook weight: %s", value)
 		}
 		if weight < HookMinWeight || weight > HookMaxWeight {
 			return nil, fmt.Errorf("invalid hook weight: %d (allowed range: %d..%d)", weight, HookMinWeight, HookMaxWeight)

internal/templatex/functions.go

@@ -9,13 +9,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math"
 	"strconv"
 	"strings"
 	"text/template"
 
-	"github.com/pkg/errors"
 	"github.com/sap/go-generics/slices"
 	"github.com/spf13/cast"
 
@@ -232,7 +232,7 @@ func parseIPv4Address(data any) (uint32, error) {
 	}
 	octets := strings.Split(s, ".")
 	if len(octets) != 4 {
-		return 0, errors.New("invalid IP address")
+		return 0, fmt.Errorf("invalid IP address")
 	}
 	var r uint64
 	for i := uint64(0); i < 4; i++ {

internal/walk/walk.go

@@ -6,12 +6,11 @@ SPDX-License-Identifier: Apache-2.0
 package walk
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
 	"strconv"
 	"strings"
-
-	"github.com/hashicorp/go-multierror"
 )
 
 type WalkFunc func(x any, path []string, tag reflect.StructTag) error
@@ -42,7 +41,7 @@ func Walk(x any, f WalkFunc) error {
 	}
 	errs := walk(v, nil, "", f)
 	if len(errs) > 0 {
-		return multierror.Append(nil, errs...)
+		return errors.Join(errs...)
 	}
 	return nil
 }

pkg/cluster/client.go

@@ -6,12 +6,20 @@ SPDX-License-Identifier: Apache-2.0
 package cluster
 
 import (
+	"context"
 	"net/http"
+	"sync"
+	"time"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/sap/component-operator-runtime/pkg/types"
 )
 
 func NewClient(clnt client.Client, discoveryClient discovery.DiscoveryInterface, eventRecorder record.EventRecorder, config *rest.Config, httpClient *http.Client) Client {
@@ -21,15 +29,23 @@ func NewClient(clnt client.Client, discoveryClient discovery.DiscoveryInterface,
 		eventRecorder:   eventRecorder,
 		config:          config,
 		httpClient:      httpClient,
+		inflightRetries: make(map[apitypes.UID]time.Time),
 	}
 }
 
+const (
+	retryAfter         = time.Second
+	nextRetryNotBefore = time.Minute
+)
+
 type clientImpl struct {
 	client.Client
 	discoveryClient discovery.DiscoveryInterface
 	eventRecorder   record.EventRecorder
 	config          *rest.Config
 	httpClient      *http.Client
+	mu              sync.Mutex
+	inflightRetries map[apitypes.UID]time.Time
 }
 
 func (c *clientImpl) DiscoveryClient() discovery.DiscoveryInterface {
@@ -47,3 +63,70 @@ func (c *clientImpl) Config() *rest.Config {
 func (c *clientImpl) HttpClient() *http.Client {
 	return c.httpClient
 }
+
+func (c *clientImpl) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error {
+	return c.Client.Apply(ctx, obj, opts...)
+}
+
+func (c *clientImpl) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+	return c.retryIfEligible(c.Client.Create(ctx, obj, opts...), obj.GetUID())
+}
+
+func (c *clientImpl) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
+	return c.retryIfEligible(c.Client.Delete(ctx, obj, opts...), obj.GetUID())
+}
+
+func (c *clientImpl) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
+	return c.retryIfEligible(c.Client.Update(ctx, obj, opts...), obj.GetUID())
+}
+
+func (c *clientImpl) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
+	return c.retryIfEligible(c.Client.Patch(ctx, obj, patch, opts...), obj.GetUID())
+}
+
+func (c *clientImpl) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
+	return c.Client.DeleteAllOf(ctx, obj, opts...)
+}
+
+func (c *clientImpl) Status() client.SubResourceWriter {
+	return &subResourceClientImpl{SubResourceClient: c.Client.SubResource("status"), client: c}
+}
+
+func (c *clientImpl) SubResource(subResource string) client.SubResourceClient {
+	return &subResourceClientImpl{SubResourceClient: c.Client.SubResource(subResource), client: c}
+}
+
+func (c *clientImpl) retryIfEligible(err error, uid apitypes.UID) error {
+	if apierrors.IsConflict(err) && uid != "" {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		now := time.Now()
+		for uid, notBefore := range c.inflightRetries {
+			if notBefore.After(now) {
+				delete(c.inflightRetries, uid)
+			}
+		}
+		if _, ok := c.inflightRetries[uid]; !ok {
+			c.inflightRetries[uid] = now.Add(nextRetryNotBefore)
+			return types.NewRetriableError(err, ref(retryAfter))
+		}
+	}
+	return err
+}
+
+type subResourceClientImpl struct {
+	client.SubResourceClient
+	client *clientImpl
+}
+
+func (s *subResourceClientImpl) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error {
+	return s.client.retryIfEligible(s.SubResourceClient.Create(ctx, obj, subResource, opts...), obj.GetUID())
+}
+
+func (s *subResourceClientImpl) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
+	return s.client.retryIfEligible(s.SubResourceClient.Update(ctx, obj, opts...), obj.GetUID())
+}
+
+func (s *subResourceClientImpl) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
+	return s.client.retryIfEligible(s.SubResourceClient.Patch(ctx, obj, patch, opts...), obj.GetUID())
+}

pkg/cluster/util.go

@@ -0,0 +1,12 @@
+/*
+SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and component-operator-runtime contributors
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package cluster
+
+// TODO: consolidate all the util files into an internal reuse package
+
+func ref[T any](x T) *T {
+	return &x
+}