Skip to content

Commit c58f1b3

Browse files
kcp-ci-botkcp-ci-bot
authored
Merge pull request #4085 from ntnn/workspace-recover-from-unavailable
Recover workspaces from Unavailable
2 parents ef9dbf4 + c7d5e64 commit c58f1b3

2 files changed

Lines changed: 138 additions & 1 deletion

File tree

‎pkg/reconciler/tenancy/workspace/workspace_reconcile_phase.go‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ type phaseReconciler struct {
3939
requeueAfter func(workspace *tenancyv1alpha1.Workspace, after time.Duration)
4040
}
4141

42+
// logicalClusterTimeout is the time limit for a logical cluster to
43+
// appear for the shard before marking its corresponding Workspace as
44+
// Unavailable.
45+
const logicalClusterTimeout = time.Second
46+
4247
func (r *phaseReconciler) reconcile(ctx context.Context, workspace *tenancyv1alpha1.Workspace) (reconcileStatus, error) {
4348
logger := klog.FromContext(ctx).WithValues("reconciler", "phase")
4449

@@ -61,7 +66,7 @@ func (r *phaseReconciler) reconcile(ctx context.Context, workspace *tenancyv1alp
6166
if hasShard {
6267
shard, shardErr := r.getShardByHash(shardHash)
6368
if shardErr == nil && shard.DeletionTimestamp.IsZero() {
64-
if workspace.CreationTimestamp.IsZero() || time.Since(workspace.CreationTimestamp.Time) < 10*time.Second {
69+
if workspace.CreationTimestamp.IsZero() || time.Since(workspace.CreationTimestamp.Time) < logicalClusterTimeout {
6570
logger.V(3).Info("LogicalCluster not found but shard is alive, requeueing", "shard", shard.Name)
6671
r.requeueAfter(workspace, 1*time.Second)
6772
return reconcileStatusContinue, nil
@@ -97,6 +102,27 @@ func (r *phaseReconciler) reconcile(ctx context.Context, workspace *tenancyv1alp
97102
conditions.MarkTrue(workspace, tenancyv1alpha1.WorkspaceInitialized)
98103

99104
case corev1alpha1.LogicalClusterPhaseUnavailable:
105+
if conditions.IsFalse(workspace, tenancyv1alpha1.WorkspaceInitialized) && conditions.GetReason(workspace, tenancyv1alpha1.WorkspaceInitialized) == tenancyv1alpha1.WorkspaceInitializedWorkspaceDisappeared {
106+
shardHash, hasShard := workspace.Annotations[WorkspaceShardHashAnnotationKey]
107+
if hasShard {
108+
shard, shardErr := r.getShardByHash(shardHash)
109+
if shardErr == nil && shard.DeletionTimestamp.IsZero() {
110+
_, err := r.getLogicalCluster(ctx, logicalcluster.NewPath(workspace.Spec.Cluster))
111+
if apierrors.IsNotFound(err) {
112+
return reconcileStatusStopAndRequeue, nil
113+
}
114+
if err != nil {
115+
return reconcileStatusStopAndRequeue, err
116+
}
117+
// Only resetting the condition. Terminal condition phase will update the phase and the
118+
// next reconcile will continue where the workspace left off.
119+
logger.V(3).Info("LogicalCluster reappeared, recovering workspace", "cluster", workspace.Spec.Cluster)
120+
conditions.MarkTrue(workspace, tenancyv1alpha1.WorkspaceInitialized)
121+
// Immediately request requeueing to recover the workspace faster.
122+
r.requeueAfter(workspace, time.Second)
123+
}
124+
}
125+
}
100126
if updateTerminalConditionPhase(workspace) {
101127
return reconcileStatusStopAndRequeue, nil
102128
}

‎pkg/reconciler/tenancy/workspace/workspace_reconcile_phase_test.go‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,117 @@ func TestReconcilePhase(t *testing.T) {
302302
wantPhase: corev1alpha1.LogicalClusterPhaseReady,
303303
wantStatus: reconcileStatusStopAndRequeue,
304304
},
305+
{
306+
name: "workspace is unavailable due to disappeared logical cluster and shard is alive - recovers when LC reappears",
307+
input: &tenancyv1alpha1.Workspace{
308+
ObjectMeta: metav1.ObjectMeta{
309+
Annotations: map[string]string{
310+
WorkspaceShardHashAnnotationKey: "shard-hash-1",
311+
},
312+
},
313+
Spec: tenancyv1alpha1.WorkspaceSpec{
314+
URL: "http://example.com",
315+
Cluster: "cluster-1",
316+
},
317+
Status: tenancyv1alpha1.WorkspaceStatus{
318+
Phase: corev1alpha1.LogicalClusterPhaseUnavailable,
319+
Conditions: []conditionsv1alpha1.Condition{
320+
{
321+
Type: tenancyv1alpha1.WorkspaceInitialized,
322+
Status: corev1.ConditionFalse,
323+
Severity: conditionsv1alpha1.ConditionSeverityError,
324+
Reason: tenancyv1alpha1.WorkspaceInitializedWorkspaceDisappeared,
325+
Message: "LogicalCluster cluster-1 not found",
326+
},
327+
},
328+
},
329+
},
330+
getLogicalCluster: func(ctx context.Context, cluster logicalcluster.Path) (*corev1alpha1.LogicalCluster, error) {
331+
return &corev1alpha1.LogicalCluster{
332+
Status: corev1alpha1.LogicalClusterStatus{
333+
Initializers: []corev1alpha1.LogicalClusterInitializer{},
334+
},
335+
}, nil
336+
},
337+
getShardByHash: func(hash string) (*corev1alpha1.Shard, error) {
338+
return &corev1alpha1.Shard{ObjectMeta: metav1.ObjectMeta{Name: "shard-1"}}, nil
339+
},
340+
wantPhase: corev1alpha1.LogicalClusterPhaseReady,
341+
wantStatus: reconcileStatusStopAndRequeue,
342+
wantCondition: conditionsv1alpha1.Condition{
343+
Type: tenancyv1alpha1.WorkspaceInitialized,
344+
Status: corev1.ConditionTrue,
345+
},
346+
wantRequeue: true,
347+
},
348+
{
349+
name: "workspace is unavailable due to disappeared logical cluster and shard is alive - requeues when LC still not found",
350+
input: &tenancyv1alpha1.Workspace{
351+
ObjectMeta: metav1.ObjectMeta{
352+
Annotations: map[string]string{
353+
WorkspaceShardHashAnnotationKey: "shard-hash-1",
354+
},
355+
},
356+
Spec: tenancyv1alpha1.WorkspaceSpec{
357+
URL: "http://example.com",
358+
Cluster: "cluster-1",
359+
},
360+
Status: tenancyv1alpha1.WorkspaceStatus{
361+
Phase: corev1alpha1.LogicalClusterPhaseUnavailable,
362+
Conditions: []conditionsv1alpha1.Condition{
363+
{
364+
Type: tenancyv1alpha1.WorkspaceInitialized,
365+
Status: corev1.ConditionFalse,
366+
Severity: conditionsv1alpha1.ConditionSeverityError,
367+
Reason: tenancyv1alpha1.WorkspaceInitializedWorkspaceDisappeared,
368+
Message: "LogicalCluster cluster-1 not found",
369+
},
370+
},
371+
},
372+
},
373+
getLogicalCluster: func(ctx context.Context, cluster logicalcluster.Path) (*corev1alpha1.LogicalCluster, error) {
374+
return nil, apierrors.NewNotFound(corev1alpha1.Resource("logicalcluster"), "cluster-1")
375+
},
376+
getShardByHash: func(hash string) (*corev1alpha1.Shard, error) {
377+
return &corev1alpha1.Shard{ObjectMeta: metav1.ObjectMeta{Name: "shard-1"}}, nil
378+
},
379+
wantPhase: corev1alpha1.LogicalClusterPhaseUnavailable,
380+
wantStatus: reconcileStatusStopAndRequeue,
381+
},
382+
{
383+
name: "workspace is unavailable due to disappeared logical cluster and shard is gone - no recovery",
384+
input: &tenancyv1alpha1.Workspace{
385+
ObjectMeta: metav1.ObjectMeta{
386+
Annotations: map[string]string{
387+
WorkspaceShardHashAnnotationKey: "shard-hash-1",
388+
},
389+
},
390+
Spec: tenancyv1alpha1.WorkspaceSpec{
391+
URL: "http://example.com",
392+
Cluster: "cluster-1",
393+
},
394+
Status: tenancyv1alpha1.WorkspaceStatus{
395+
Phase: corev1alpha1.LogicalClusterPhaseUnavailable,
396+
Conditions: []conditionsv1alpha1.Condition{
397+
{
398+
Type: tenancyv1alpha1.WorkspaceInitialized,
399+
Status: corev1.ConditionFalse,
400+
Severity: conditionsv1alpha1.ConditionSeverityError,
401+
Reason: tenancyv1alpha1.WorkspaceInitializedWorkspaceDisappeared,
402+
Message: "LogicalCluster cluster-1 not found",
403+
},
404+
},
405+
},
406+
},
407+
getLogicalCluster: func(ctx context.Context, cluster logicalcluster.Path) (*corev1alpha1.LogicalCluster, error) {
408+
return nil, apierrors.NewNotFound(corev1alpha1.Resource("logicalcluster"), "cluster-1")
409+
},
410+
getShardByHash: func(hash string) (*corev1alpha1.Shard, error) {
411+
return nil, apierrors.NewNotFound(corev1alpha1.Resource("shard"), "shard-hash-1")
412+
},
413+
wantPhase: corev1alpha1.LogicalClusterPhaseUnavailable,
414+
wantStatus: reconcileStatusContinue,
415+
},
305416
} {
306417
t.Run(testCase.name, func(t *testing.T) {
307418
requeued := false

0 commit comments

Comments
 (0)