Skip to content

Commit 1a24ec8

Browse files
committed
fix: stabilize InferencePool parent status reconciliation
Fixes #1331. Inference Extension conformance can create multiple HTTPRoutes for the same InferencePool in quick succession. A reconcile can observe only one referenced Gateway through the controller cache and write a partial Parents status, then receive no later event to recompute the complete parent set. Return a short requeue only when the InferencePool status actually changes. Status equality ignores LastTransitionTime so the follow-up reconcile converges and stops instead of looping forever. Constraint: controller-runtime cache-backed route and gateway lists can lag related object events. Rejected: Increasing conformance timeouts, because stale status can persist indefinitely without another reconcile. Confidence: high Scope-risk: moderate; changes InferencePool reconcile scheduling after status updates. Tested: go test ./internal/controller -run TestInferencePoolController Tested: go test ./internal/controller Tested: make lint Signed-off-by: liuhy <liuhongyu@apache.org>
1 parent 2451a73 commit 1a24ec8

2 files changed

Lines changed: 352 additions & 7 deletions

File tree

internal/controller/inference_pool.go

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"context"
1010
"fmt"
1111
"strings"
12+
"time"
1213

1314
"github.com/go-logr/logr"
1415
corev1 "k8s.io/api/core/v1"
@@ -24,6 +25,8 @@ import (
2425
aigv1b1 "github.com/envoyproxy/ai-gateway/api/v1beta1"
2526
)
2627

28+
const inferencePoolStatusRequeueAfter = time.Second
29+
2730
// InferencePoolController implements [reconcile.TypedReconciler] for [gwaiev1.InferencePool].
2831
//
2932
// This handles the InferencePool resource and updates its status based on associated Gateways.
@@ -67,7 +70,9 @@ func (c *InferencePoolController) Reconcile(ctx context.Context, req reconcile.R
6770
c.updateInferencePoolStatus(ctx, &inferencePool, "NotAccepted", err.Error())
6871
return ctrl.Result{}, err
6972
}
70-
c.updateInferencePoolStatus(ctx, &inferencePool, "Accepted", "InferencePool reconciled successfully")
73+
if c.updateInferencePoolStatus(ctx, &inferencePool, "Accepted", "InferencePool reconciled successfully") {
74+
return ctrl.Result{RequeueAfter: inferencePoolStatusRequeueAfter}, nil
75+
}
7176
return ctrl.Result{}, nil
7277
}
7378

@@ -227,16 +232,18 @@ func (c *InferencePoolController) httpRouteReferencesInferencePool(route *gwapiv
227232
return false
228233
}
229234

230-
// updateInferencePoolStatus updates the status of the InferencePool.
231-
func (c *InferencePoolController) updateInferencePoolStatus(ctx context.Context, inferencePool *gwaiev1.InferencePool, conditionType string, message string) {
235+
// updateInferencePoolStatus updates the status of the InferencePool and reports
236+
// whether the status changed. Requeueing after a change lets cache-backed route
237+
// and gateway lists converge before conformance checks observe parent status.
238+
func (c *InferencePoolController) updateInferencePoolStatus(ctx context.Context, inferencePool *gwaiev1.InferencePool, conditionType string, message string) bool {
232239
// Check if this is an ExtensionReference validation error.
233240
isExtensionRefError := conditionType == "NotAccepted" &&
234241
(strings.Contains(message, "ExtensionReference service") && strings.Contains(message, "not found"))
235242
// Get the referenced Gateways from syncInferencePool logic.
236243
referencedGateways, err := c.getReferencedGateways(ctx, inferencePool)
237244
if err != nil {
238245
c.logger.Error(err, "failed to get referenced Gateways for status update")
239-
return
246+
return false
240247
}
241248

242249
// Build Parents status.
@@ -278,10 +285,69 @@ func (c *InferencePoolController) updateInferencePoolStatus(ctx context.Context,
278285
// If no Gateways reference this InferencePool, clear all parents.
279286
// This correctly reflects that the InferencePool is not currently referenced by any Gateway.
280287

288+
if inferencePoolParentStatusesEqual(inferencePool.Status.Parents, parents) {
289+
return false
290+
}
291+
281292
inferencePool.Status.Parents = parents
282293
if err := c.client.Status().Update(ctx, inferencePool); err != nil {
283294
c.logger.Error(err, "failed to update InferencePool status")
295+
return false
296+
}
297+
return true
298+
}
299+
300+
func inferencePoolParentStatusesEqual(a, b []gwaiev1.ParentStatus) bool {
301+
if len(a) != len(b) {
302+
return false
303+
}
304+
305+
remaining := make(map[string]gwaiev1.ParentStatus, len(b))
306+
for i := range b {
307+
parent := &b[i]
308+
remaining[inferencePoolParentStatusKey(parent)] = *parent
309+
}
310+
for i := range a {
311+
parent := &a[i]
312+
other, ok := remaining[inferencePoolParentStatusKey(parent)]
313+
if !ok || !inferencePoolConditionsEqual(parent.Conditions, other.Conditions) {
314+
return false
315+
}
316+
delete(remaining, inferencePoolParentStatusKey(parent))
317+
}
318+
return len(remaining) == 0
319+
}
320+
321+
func inferencePoolParentStatusKey(parent *gwaiev1.ParentStatus) string {
322+
ref := parent.ParentRef
323+
group := ""
324+
if ref.Group != nil {
325+
group = string(*ref.Group)
326+
}
327+
return fmt.Sprintf("%s/%s/%s/%s", group, ref.Kind, ref.Namespace, ref.Name)
328+
}
329+
330+
func inferencePoolConditionsEqual(a, b []metav1.Condition) bool {
331+
if len(a) != len(b) {
332+
return false
333+
}
334+
335+
remaining := make(map[string]metav1.Condition, len(b))
336+
for _, condition := range b {
337+
remaining[condition.Type] = condition
338+
}
339+
for _, condition := range a {
340+
other, ok := remaining[condition.Type]
341+
if !ok ||
342+
condition.Status != other.Status ||
343+
condition.Reason != other.Reason ||
344+
condition.Message != other.Message ||
345+
condition.ObservedGeneration != other.ObservedGeneration {
346+
return false
347+
}
348+
delete(remaining, condition.Type)
284349
}
350+
return len(remaining) == 0
285351
}
286352

287353
// buildAcceptedCondition builds a condition for the InferencePool status.

0 commit comments

Comments
 (0)