Skip to content

Commit 7012b18

Browse files
freeznetclaude
andauthored
fix(connection): use MergeFrom patch for finalizer writes to avoid 409 conflicts (#414)
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent b8f7a8b commit 7012b18

17 files changed

Lines changed: 471 additions & 65 deletions

controllers/pulsarconnection_controller.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ import (
4343
// PulsarConnectionReconciler reconciles a PulsarConnection object
4444
type PulsarConnectionReconciler struct {
4545
client.Client
46+
// APIReader is an uncached reader (mgr.GetAPIReader()) used for finalizer reads that must
47+
// observe a fresh resourceVersion, so deletion is not wedged by informer-cache lag.
48+
APIReader client.Reader
4649
Scheme *runtime.Scheme
4750
Log logr.Logger
4851
Recorder record.EventRecorder
@@ -112,7 +115,7 @@ func (r *PulsarConnectionReconciler) Reconcile(ctx context.Context, req ctrl.Req
112115

113116
r.Log.Info("Reconciling PulsarConnection", "name", pulsarConnection.Name, "namespace", pulsarConnection.Namespace)
114117

115-
reconciler := connection.MakeReconciler(r.Log, r.Client, r.PulsarAdminCreator, pulsarConnection, r.Retryer)
118+
reconciler := connection.MakeReconciler(r.Log, r.Client, r.APIReader, r.PulsarAdminCreator, pulsarConnection, r.Retryer)
116119
if err := reconciler.Observe(ctx); err != nil {
117120
return ctrl.Result{}, err
118121
}

main.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ func main() {
120120

121121
if err = (&controllers.PulsarConnectionReconciler{
122122
Client: mgr.GetClient(),
123+
APIReader: mgr.GetAPIReader(),
123124
Scheme: mgr.GetScheme(),
124125
Log: ctrl.Log.WithName("controllers").WithName("PulsarConnection"),
125126
Recorder: mgr.GetEventRecorderFor("pulsarconnection-controller"),
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Copyright 2025 StreamNative
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package connection
16+
17+
import (
18+
"context"
19+
20+
"k8s.io/client-go/util/retry"
21+
"sigs.k8s.io/controller-runtime/pkg/client"
22+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
23+
)
24+
25+
// ensureFinalizer adds finalizer to obj, persisting the change against the live API object
26+
// and retrying on optimistic-lock conflicts.
27+
//
28+
// reader must be an uncached reader (e.g. mgr.GetAPIReader()): the manager's default client
29+
// reads through the informer cache, which can lag behind a recent write and hand back a stale
30+
// resourceVersion, causing the Update to 409 and the retry to keep re-reading the same stale
31+
// copy. Reading live guarantees a fresh resourceVersion so the write succeeds on the first
32+
// reconcile regardless of cache lag. Writes use the normal writer client.
33+
//
34+
// Guard: when the cached copy of obj already contains finalizer this returns immediately with
35+
// no API calls, so the common steady-state path (finalizer already present) pays nothing —
36+
// only the first add issues the uncached read.
37+
//
38+
// Only this operator's finalizer is mutated, so finalizers added concurrently by other
39+
// controllers are preserved. On success obj holds the server's latest state (including the
40+
// refreshed resourceVersion), so callers may safely issue a follow-up Status().Update.
41+
func ensureFinalizer(ctx context.Context, reader client.Reader, writer client.Client, obj client.Object, finalizer string) error {
42+
if controllerutil.ContainsFinalizer(obj, finalizer) {
43+
return nil
44+
}
45+
key := client.ObjectKeyFromObject(obj)
46+
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
47+
if err := reader.Get(ctx, key, obj); err != nil {
48+
return err
49+
}
50+
if !controllerutil.AddFinalizer(obj, finalizer) {
51+
// Already present on the live object; nothing to write.
52+
return nil
53+
}
54+
return writer.Update(ctx, obj)
55+
})
56+
}
57+
58+
// removeFinalizer removes finalizer from obj, persisting the change against the live API
59+
// object and retrying on optimistic-lock conflicts.
60+
//
61+
// reader must be an uncached reader (e.g. mgr.GetAPIReader()). Re-reading the live object
62+
// before each attempt yields a fresh resourceVersion, so deletion is not wedged by a stale
63+
// informer cache (the original stuck-Terminating symptom), and only this operator's finalizer
64+
// is removed — finalizers owned by other controllers are preserved and the object is not
65+
// garbage-collected prematurely. Writes use the normal writer client.
66+
//
67+
// Guard: when the cached copy of obj does not contain finalizer this returns immediately with
68+
// no API calls. A NotFound result (the object was garbage-collected once its last finalizer
69+
// was removed) is treated as success.
70+
func removeFinalizer(ctx context.Context, reader client.Reader, writer client.Client, obj client.Object, finalizer string) error {
71+
if !controllerutil.ContainsFinalizer(obj, finalizer) {
72+
return nil
73+
}
74+
key := client.ObjectKeyFromObject(obj)
75+
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
76+
if err := reader.Get(ctx, key, obj); err != nil {
77+
return err
78+
}
79+
if !controllerutil.RemoveFinalizer(obj, finalizer) {
80+
// Already absent on the live object; nothing to write.
81+
return nil
82+
}
83+
return writer.Update(ctx, obj)
84+
})
85+
return client.IgnoreNotFound(err)
86+
}
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright 2026 StreamNative
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package connection
16+
17+
import (
18+
"context"
19+
"testing"
20+
21+
apierrors "k8s.io/apimachinery/pkg/api/errors"
22+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
23+
"sigs.k8s.io/controller-runtime/pkg/client"
24+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
25+
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
26+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
27+
28+
resourcev1alpha1 "github.com/streamnative/pulsar-resources-operator/api/v1alpha1"
29+
)
30+
31+
// TestRemoveFinalizerUsesLiveReaderResourceVersion proves the finalizer write is driven by the
32+
// reader, not by the caller's (possibly stale) in-hand object. A reader that keeps returning a
33+
// stale resourceVersion — as the manager's cached client can under informer lag — makes every
34+
// retry 409, so the removal fails (the original stuck-Terminating symptom). The same removal
35+
// with a live reader reads a fresh resourceVersion and succeeds. In production the reader is the
36+
// uncached mgr.GetAPIReader(), so a lagging informer cache cannot wedge deletion.
37+
//
38+
// This is the case ordinary single-fake-client tests cannot exercise: here the reader and the
39+
// write store are deliberately decoupled to simulate cache lag.
40+
func TestRemoveFinalizerUsesLiveReaderResourceVersion(t *testing.T) {
41+
scheme := newSourceTestScheme(t)
42+
topic := &resourcev1alpha1.PulsarTopic{
43+
ObjectMeta: metav1.ObjectMeta{
44+
Name: "t",
45+
Namespace: "ns",
46+
Finalizers: []string{resourcev1alpha1.FinalizerName},
47+
},
48+
}
49+
writer := fake.NewClientBuilder().WithScheme(scheme).WithObjects(topic).Build()
50+
key := client.ObjectKeyFromObject(topic)
51+
52+
// Snapshot the object at its current resourceVersion, then advance the live RV out of band.
53+
staleObj := &resourcev1alpha1.PulsarTopic{}
54+
if err := writer.Get(context.Background(), key, staleObj); err != nil {
55+
t.Fatalf("get snapshot: %v", err)
56+
}
57+
bumped := staleObj.DeepCopy()
58+
bumped.Annotations = map[string]string{"bump": "1"}
59+
if err := writer.Update(context.Background(), bumped); err != nil {
60+
t.Fatalf("bump live resourceVersion: %v", err)
61+
}
62+
63+
// staleReader always returns the pre-bump snapshot — a persistently stale resourceVersion.
64+
staleReader := interceptor.NewClient(writer, interceptor.Funcs{
65+
Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, obj client.Object, _ ...client.GetOption) error {
66+
staleObj.DeepCopyInto(obj.(*resourcev1alpha1.PulsarTopic))
67+
return nil
68+
},
69+
})
70+
71+
// Persistently stale reader: every retry re-reads the stale RV and the Update conflicts.
72+
if err := removeFinalizer(context.Background(), staleReader, writer, staleObj.DeepCopy(),
73+
resourcev1alpha1.FinalizerName); !apierrors.IsConflict(err) {
74+
t.Fatalf("expected a 409 conflict from a persistently stale reader, got: %v", err)
75+
}
76+
77+
// Live (uncached) reader: the removal reads a fresh RV and succeeds.
78+
if err := removeFinalizer(context.Background(), writer, writer, staleObj.DeepCopy(),
79+
resourcev1alpha1.FinalizerName); err != nil {
80+
t.Fatalf("removeFinalizer with a live reader: %v", err)
81+
}
82+
got := &resourcev1alpha1.PulsarTopic{}
83+
if err := writer.Get(context.Background(), key, got); err != nil {
84+
t.Fatalf("get after removal: %v", err)
85+
}
86+
if controllerutil.ContainsFinalizer(got, resourcev1alpha1.FinalizerName) {
87+
t.Fatalf("operator finalizer should have been removed, got %v", got.Finalizers)
88+
}
89+
}
90+
91+
// TestEnsureFinalizerGuardSkipsReadWhenPresent verifies the steady-state fast path: when the
92+
// cached object already carries the finalizer, ensureFinalizer issues no API calls at all, so
93+
// routing the read through the uncached API reader adds no per-reconcile apiserver load on the
94+
// common path.
95+
func TestEnsureFinalizerGuardSkipsReadWhenPresent(t *testing.T) {
96+
scheme := newSourceTestScheme(t)
97+
writer := fake.NewClientBuilder().WithScheme(scheme).Build()
98+
panicReader := interceptor.NewClient(writer, interceptor.Funcs{
99+
Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error {
100+
panic("reader.Get must not be called when the finalizer is already present")
101+
},
102+
})
103+
104+
obj := &resourcev1alpha1.PulsarTopic{
105+
ObjectMeta: metav1.ObjectMeta{
106+
Name: "t",
107+
Namespace: "ns",
108+
Finalizers: []string{resourcev1alpha1.FinalizerName},
109+
},
110+
}
111+
if err := ensureFinalizer(context.Background(), panicReader, writer, obj, resourcev1alpha1.FinalizerName); err != nil {
112+
t.Fatalf("ensureFinalizer (guard fast path) returned error: %v", err)
113+
}
114+
}

pkg/connection/reconcile_function.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525
"github.com/streamnative/pulsar-resources-operator/pkg/feature"
2626
"k8s.io/apimachinery/pkg/api/meta"
2727
"sigs.k8s.io/controller-runtime/pkg/client"
28-
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2928

3029
resourcev1alpha1 "github.com/streamnative/pulsar-resources-operator/api/v1alpha1"
3130
"github.com/streamnative/pulsar-resources-operator/pkg/admin"
@@ -101,8 +100,7 @@ func (r *PulsarFunctionReconciler) ReconcileFunction(ctx context.Context, pulsar
101100
}
102101

103102
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
104-
controllerutil.RemoveFinalizer(instance, resourcev1alpha1.FinalizerName)
105-
if err := r.conn.client.Update(ctx, instance); err != nil {
103+
if err := removeFinalizer(ctx, r.conn.apiReader, r.conn.client, instance, resourcev1alpha1.FinalizerName); err != nil {
106104
log.Error(err, "Failed to remove finalizer")
107105
return err
108106
}
@@ -111,8 +109,7 @@ func (r *PulsarFunctionReconciler) ReconcileFunction(ctx context.Context, pulsar
111109

112110
if instance.Spec.LifecyclePolicy != resourcev1alpha1.KeepAfterDeletion {
113111
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
114-
controllerutil.AddFinalizer(instance, resourcev1alpha1.FinalizerName)
115-
if err := r.conn.client.Update(ctx, instance); err != nil {
112+
if err := ensureFinalizer(ctx, r.conn.apiReader, r.conn.client, instance, resourcev1alpha1.FinalizerName); err != nil {
116113
log.Error(err, "Failed to add finalizer")
117114
return err
118115
}

pkg/connection/reconcile_geo_replication.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727
"k8s.io/apimachinery/pkg/api/meta"
2828
"k8s.io/apimachinery/pkg/types"
2929
"sigs.k8s.io/controller-runtime/pkg/client"
30-
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
3130

3231
resourcev1alpha1 "github.com/streamnative/pulsar-resources-operator/api/v1alpha1"
3332
"github.com/streamnative/pulsar-resources-operator/pkg/admin"
@@ -168,15 +167,13 @@ func (r *PulsarGeoReplicationReconciler) ReconcileGeoReplication(ctx context.Con
168167
}
169168
}
170169
}
171-
controllerutil.RemoveFinalizer(geoReplication, resourcev1alpha1.FinalizerName)
172-
if err := r.conn.client.Update(ctx, geoReplication); err != nil {
170+
if err := removeFinalizer(ctx, r.conn.apiReader, r.conn.client, geoReplication, resourcev1alpha1.FinalizerName); err != nil {
173171
log.Error(err, "Failed to remove finalizer")
174172
return err
175173
}
176174
}
177175
}
178-
controllerutil.AddFinalizer(geoReplication, resourcev1alpha1.FinalizerName)
179-
if err := r.conn.client.Update(ctx, geoReplication); err != nil {
176+
if err := ensureFinalizer(ctx, r.conn.apiReader, r.conn.client, geoReplication, resourcev1alpha1.FinalizerName); err != nil {
180177
log.Error(err, "Failed to add finalizer")
181178
return err
182179
}

pkg/connection/reconcile_namespace.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525
"k8s.io/apimachinery/pkg/api/meta"
2626
"k8s.io/apimachinery/pkg/types"
2727
"sigs.k8s.io/controller-runtime/pkg/client"
28-
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2928

3029
resourcev1alpha1 "github.com/streamnative/pulsar-resources-operator/api/v1alpha1"
3130
"github.com/streamnative/pulsar-resources-operator/pkg/admin"
@@ -124,8 +123,7 @@ func (r *PulsarNamespaceReconciler) ReconcileNamespace(ctx context.Context, puls
124123
}
125124

126125
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
127-
controllerutil.RemoveFinalizer(namespace, resourcev1alpha1.FinalizerName)
128-
if err := r.conn.client.Update(ctx, namespace); err != nil {
126+
if err := removeFinalizer(ctx, r.conn.apiReader, r.conn.client, namespace, resourcev1alpha1.FinalizerName); err != nil {
129127
log.Error(err, "Failed to remove finalizer")
130128
return err
131129
}
@@ -135,8 +133,7 @@ func (r *PulsarNamespaceReconciler) ReconcileNamespace(ctx context.Context, puls
135133

136134
if namespace.Spec.LifecyclePolicy != resourcev1alpha1.KeepAfterDeletion {
137135
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
138-
controllerutil.AddFinalizer(namespace, resourcev1alpha1.FinalizerName)
139-
if err := r.conn.client.Update(ctx, namespace); err != nil {
136+
if err := ensureFinalizer(ctx, r.conn.apiReader, r.conn.client, namespace, resourcev1alpha1.FinalizerName); err != nil {
140137
log.Error(err, "Failed to add finalizer")
141138
return err
142139
}

pkg/connection/reconcile_nsisolationpolicy.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"github.com/streamnative/pulsar-resources-operator/pkg/reconciler"
2727
"k8s.io/apimachinery/pkg/api/meta"
2828
"sigs.k8s.io/controller-runtime/pkg/client"
29-
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
3029
)
3130

3231
// PulsarNSIsolationPolicyReconciler reconciles a PulsarNSIsolationPolicy object
@@ -105,8 +104,7 @@ func (r *PulsarNSIsolationPolicyReconciler) ReconcilePolicy(ctx context.Context,
105104
}
106105

107106
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
108-
controllerutil.RemoveFinalizer(policy, resourcev1alpha1.FinalizerName)
109-
if err := r.conn.client.Update(ctx, policy); err != nil {
107+
if err := removeFinalizer(ctx, r.conn.apiReader, r.conn.client, policy, resourcev1alpha1.FinalizerName); err != nil {
110108
log.Error(err, "Failed to remove finalizer")
111109
return err
112110
}
@@ -115,8 +113,7 @@ func (r *PulsarNSIsolationPolicyReconciler) ReconcilePolicy(ctx context.Context,
115113
}
116114

117115
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
118-
controllerutil.AddFinalizer(policy, resourcev1alpha1.FinalizerName)
119-
if err := r.conn.client.Update(ctx, policy); err != nil {
116+
if err := ensureFinalizer(ctx, r.conn.apiReader, r.conn.client, policy, resourcev1alpha1.FinalizerName); err != nil {
120117
log.Error(err, "Failed to add finalizer")
121118
return err
122119
}

pkg/connection/reconcile_package.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import (
2727
"github.com/streamnative/pulsar-resources-operator/pkg/feature"
2828
"k8s.io/apimachinery/pkg/api/meta"
2929
"sigs.k8s.io/controller-runtime/pkg/client"
30-
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
3130

3231
resourcev1alpha1 "github.com/streamnative/pulsar-resources-operator/api/v1alpha1"
3332
"github.com/streamnative/pulsar-resources-operator/pkg/admin"
@@ -154,17 +153,15 @@ func (r *PulsarPackageReconciler) ReconcilePackage(ctx context.Context, pulsarAd
154153
}
155154
}
156155

157-
controllerutil.RemoveFinalizer(pkg, resourcev1alpha1.FinalizerName)
158-
if err := r.conn.client.Update(ctx, pkg); err != nil {
156+
if err := removeFinalizer(ctx, r.conn.apiReader, r.conn.client, pkg, resourcev1alpha1.FinalizerName); err != nil {
159157
log.Error(err, "Failed to remove finalizer")
160158
return err
161159
}
162160
return nil
163161
}
164162

165163
if pkg.Spec.LifecyclePolicy != resourcev1alpha1.KeepAfterDeletion {
166-
controllerutil.AddFinalizer(pkg, resourcev1alpha1.FinalizerName)
167-
if err := r.conn.client.Update(ctx, pkg); err != nil {
164+
if err := ensureFinalizer(ctx, r.conn.apiReader, r.conn.client, pkg, resourcev1alpha1.FinalizerName); err != nil {
168165
log.Error(err, "Failed to add finalizer")
169166
return err
170167
}

pkg/connection/reconcile_permission.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424
"github.com/go-logr/logr"
2525
"k8s.io/apimachinery/pkg/api/meta"
2626
"sigs.k8s.io/controller-runtime/pkg/client"
27-
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
2827

2928
resourcev1alpha1 "github.com/streamnative/pulsar-resources-operator/api/v1alpha1"
3029
"github.com/streamnative/pulsar-resources-operator/pkg/admin"
@@ -102,8 +101,7 @@ func (r *PulsarPermissionReconciler) ReconcilePermission(ctx context.Context, pu
102101
}
103102
}
104103
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
105-
controllerutil.RemoveFinalizer(permission, resourcev1alpha1.FinalizerName)
106-
if err := r.conn.client.Update(ctx, permission); err != nil {
104+
if err := removeFinalizer(ctx, r.conn.apiReader, r.conn.client, permission, resourcev1alpha1.FinalizerName); err != nil {
107105
log.Error(err, "Failed to remove finalizer")
108106
return err
109107
}
@@ -112,8 +110,7 @@ func (r *PulsarPermissionReconciler) ReconcilePermission(ctx context.Context, pu
112110

113111
if permission.Spec.LifecyclePolicy != resourcev1alpha1.KeepAfterDeletion {
114112
// TODO use otelcontroller until kube-instrumentation upgrade controller-runtime version to newer
115-
controllerutil.AddFinalizer(permission, resourcev1alpha1.FinalizerName)
116-
if err := r.conn.client.Update(ctx, permission); err != nil {
113+
if err := ensureFinalizer(ctx, r.conn.apiReader, r.conn.client, permission, resourcev1alpha1.FinalizerName); err != nil {
117114
log.Error(err, "Failed to add finalizer")
118115
return err
119116
}

0 commit comments

Comments
 (0)