Skip to content

fix: prevent stale ready status after stack update #3610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zreigz wants to merge 4 commits into
base: master
Choose a base branch
from
+72 −50
Open

fix: prevent stale ready status after stack update #3610

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
366e345
prevent stale ready status after stack create/update
zreigz May 25, 2026
c8a463a
format
zreigz May 25, 2026
ae8af25
fix status
zreigz May 25, 2026
b005557
linter, refactor
zreigz May 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion go/controller/internal/common/reconciliation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,12 @@ func HandleRequeue(result *ctrl.Result, err error, setCondition func(condition m
result = lo.ToPtr(Wait())
}

conditionReason := v1alpha1.SynchronizedConditionReason
if err != nil {
conditionReason = v1alpha1.SynchronizedConditionReasonError
}
utils.MarkCondition(setCondition, v1alpha1.SynchronizedConditionType, metav1.ConditionFalse,
v1alpha1.SynchronizedConditionReasonError, defaultErrMessage(err, ""))
conditionReason, defaultErrMessage(err, ""))
return lo.FromPtr(result), lo.Ternary(result != nil, nil, err)
}

Expand Down
110 changes: 64 additions & 46 deletions go/controller/internal/controller/infrastructurestack_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,45 +119,11 @@ func (r *InfrastructureStackReconciler) Process(ctx context.Context, req ctrl.Re
return common.HandleRequeue(result, err, stack.SetCondition)
}

clusterID, result, err := r.handleClusterRef(ctx, stack)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}

repositoryID, result, err := r.handleRepositoryRef(ctx, stack)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}

policyEngineRepositoryID, result, err := r.handlePolicyEngineRepositoryRef(ctx, stack)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}

project, result, err := common.Project(ctx, r.Client, r.Scheme, stack)
attributes, result, err := r.getDynamicAttributes(ctx, stack)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}

stackDefinitionID, result, err := r.handleStackDefinitionRef(ctx, stack)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}

metrics, result, err := r.handleObservableMetrics(ctx, stack)
if result != nil || err != nil {
return lo.FromPtr(result), err
}

attributes := dynamicAttributes{
clusterID: clusterID,
repositoryID: repositoryID,
policyEngineRepositoryID: policyEngineRepositoryID,
projectID: project.Status.ID,
definitionID: stackDefinitionID,
observableMetrics: metrics,
}

// Check if resource already exists in the API and only sync the ID
existing, err := r.stackExists(ctx, stack)
if err != nil {
Expand Down Expand Up @@ -194,7 +160,9 @@ func (r *InfrastructureStackReconciler) Process(ctx context.Context, req ctrl.Re
stack.Status.SHA = lo.ToPtr(sha)
}

changed := false
if !stack.Status.IsSHAEqual(sha) {
changed = true
_, err = r.ConsoleClient.UpdateStack(ctx, stack.Status.GetID(), *attr)
if err != nil {
utils.MarkCondition(stack.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionFalse, v1alpha1.SynchronizedConditionReasonError, err.Error())
Expand All @@ -205,38 +173,88 @@ func (r *InfrastructureStackReconciler) Process(ctx context.Context, req ctrl.Re
stack.Status.SHA = lo.ToPtr(sha)
}

if err := r.setReadyCondition(ctx, stack, existing != nil); err != nil {
return ctrl.Result{}, err
result, err = r.setReadyCondition(ctx, stack, existing != nil, changed)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}
utils.MarkCondition(stack.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionTrue, v1alpha1.SynchronizedConditionReason, "")

return ctrl.Result{RequeueAfter: requeueAfterInfrastructureStack}, nil
}

func (r *InfrastructureStackReconciler) getDynamicAttributes(ctx context.Context, stack *v1alpha1.InfrastructureStack) (*dynamicAttributes, *ctrl.Result, error) {
clusterID, result, err := r.handleClusterRef(ctx, stack)
if result != nil || err != nil {
return nil, result, err
}

repositoryID, result, err := r.handleRepositoryRef(ctx, stack)
if result != nil || err != nil {
return nil, result, err
}

policyEngineRepositoryID, result, err := r.handlePolicyEngineRepositoryRef(ctx, stack)
if result != nil || err != nil {
return nil, result, err
}

project, result, err := common.Project(ctx, r.Client, r.Scheme, stack)
if result != nil || err != nil {
return nil, result, err
}

stackDefinitionID, result, err := r.handleStackDefinitionRef(ctx, stack)
if result != nil || err != nil {
return nil, result, err
}

metrics, result, err := r.handleObservableMetrics(ctx, stack)
if result != nil || err != nil {
return nil, result, err
}

return &dynamicAttributes{
clusterID: clusterID,
repositoryID: repositoryID,
policyEngineRepositoryID: policyEngineRepositoryID,
projectID: project.Status.ID,
definitionID: stackDefinitionID,
observableMetrics: metrics,
}, nil, nil
}

func (r *InfrastructureStackReconciler) handleExistingResource(ctx context.Context, stack *v1alpha1.InfrastructureStack, apiStack *console.InfrastructureStackIDFragment) (ctrl.Result, error) {
stack.Status.ID = apiStack.ID

if err := r.setReadyCondition(ctx, stack, apiStack != nil); err != nil {
return ctrl.Result{}, err
result, err := r.setReadyCondition(ctx, stack, apiStack != nil, false)
if result != nil || err != nil {
return common.HandleRequeue(result, err, stack.SetCondition)
}

utils.MarkCondition(stack.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionTrue, v1alpha1.SynchronizedConditionReason, "")
return stack.Spec.Reconciliation.Requeue(), nil
return ctrl.Result{RequeueAfter: requeueAfterInfrastructureStack}, nil
}
Comment thread
zreigz marked this conversation as resolved.

func (r *InfrastructureStackReconciler) setReadyCondition(ctx context.Context, stack *v1alpha1.InfrastructureStack, exists bool) error {
func (r *InfrastructureStackReconciler) setReadyCondition(ctx context.Context, stack *v1alpha1.InfrastructureStack, exists, changed bool) (*ctrl.Result, error) {
if !exists {
return nil
return lo.ToPtr(common.WaitForResources()), nil
}

// wait a little bit before marking the stack as ready to give the console some time to process the updated stack
if changed {
return lo.ToPtr(ctrl.Result{RequeueAfter: v1alpha1.Jitter(5 * time.Second)}), nil
}
Comment thread
zreigz marked this conversation as resolved.
utils.MarkCondition(stack.SetCondition, v1alpha1.ReadyConditionType, v1.ConditionFalse, v1alpha1.ReadyConditionReason, "")
status, err := r.ConsoleClient.GetStackStatus(ctx, *stack.Status.ID)
if err != nil {
return err
return nil, err
}
if status.Status == console.StackStatusSuccessful {
utils.MarkCondition(stack.SetCondition, v1alpha1.ReadyConditionType, v1.ConditionTrue, v1alpha1.ReadyConditionReason, "")
return nil, nil
}
return nil

return lo.ToPtr(common.WaitForResources()), nil
}

// SetupWithManager sets up the controller with the Manager.
Expand Down Expand Up @@ -316,7 +334,7 @@ type dynamicAttributes struct {
func (r *InfrastructureStackReconciler) getStackAttributes(
ctx context.Context,
stack *v1alpha1.InfrastructureStack,
attributes dynamicAttributes,
attributes *dynamicAttributes,
) (*console.StackAttributes, error) {
attr := &console.StackAttributes{
Name: stack.StackName(),
Expand Down Expand Up @@ -670,7 +688,7 @@ func (r *InfrastructureStackReconciler) handleObservableMetrics(
if !obsProvider.Status.HasID() {
logger.Info("ObservabilityProvider not ready", "provider", key)
utils.MarkCondition(stack.SetCondition, v1alpha1.SynchronizedConditionType, v1.ConditionFalse, v1alpha1.SynchronizedConditionReason, "stack definition is not ready")
return nil, lo.ToPtr(stack.Spec.Reconciliation.Requeue()), nil
return nil, lo.ToPtr(common.WaitForResources()), nil
}

metrics = append(metrics, console.ObservableMetricAttributes{
Expand Down
6 changes: 3 additions & 3 deletions go/controller/internal/controller/infrastructurestack_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ var _ = Describe("Infrastructure Stack Controller", Ordered, func() {
},
{
Type: v1alpha1.SynchronizedConditionType.String(),
Status: metav1.ConditionTrue,
Status: metav1.ConditionFalse,
Reason: v1alpha1.SynchronizedConditionReason.String(),
},
},
Expand Down Expand Up @@ -435,12 +435,12 @@ var _ = Describe("Infrastructure Stack Controller", Ordered, func() {
},
{
Type: v1alpha1.ReadyConditionType.String(),
Status: metav1.ConditionTrue,
Status: metav1.ConditionFalse,
Reason: v1alpha1.ReadyConditionReason.String(),
},
{
Type: v1alpha1.SynchronizedConditionType.String(),
Status: metav1.ConditionTrue,
Status: metav1.ConditionFalse,
Reason: v1alpha1.SynchronizedConditionReason.String(),
},
},
Expand Down
Loading