Skip to content

Commit 4ed5020

Browse files
weicaoapecloud-bot
weicao
authored and
apecloud-bot
committed
fix(operations): rebuild no-backup instances via fresh PVCs (#10295)
(cherry picked from commit 37eb9ff)
1 parent 441fd05 commit 4ed5020

3 files changed

Lines changed: 128 additions & 27 deletions

File tree

‎pkg/operations/rebuild_instance.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -599,7 +599,7 @@ func (r rebuildInstanceOpsHandler) prepareInplaceRebuildHelper(reqCtx intctrluti
599599
return nil, err
600600
}
601601
rebuildPrefix := fmt.Sprintf("rebuild-%s", opsRes.OpsRequest.UID[:8])
602-
pvcMap, volumes, volumeMounts, err := getPVCMapAndVolumes(opsRes, synthesizedComp, targetPod, rebuildPrefix, index)
602+
pvcMap, volumes, volumeMounts, err := getPVCMapAndVolumes(opsRes, synthesizedComp, targetPod, rebuildPrefix, index, rebuildInstance.BackupName == "")
603603
if err != nil {
604604
return nil, err
605605
}

‎pkg/operations/rebuild_instance_inplace.go‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -79,17 +79,14 @@ func (inPlaceHelper *inplaceRebuildHelper) rebuildInstanceWithNoBackup(reqCtx in
7979
cli client.Client,
8080
opsRes *OpsResource,
8181
progressDetail *opsv1alpha1.ProgressStatusDetail) (bool, error) {
82-
// 1. restore the new pvs.
83-
completed, err := inPlaceHelper.rebuildInstancePVByPod(reqCtx, cli, opsRes, progressDetail)
84-
if err != nil || !completed {
85-
return false, err
86-
}
8782
if progressDetail.Message != waitingForInstanceReadyMessage {
88-
// 2. rebuild source pvcs and recreate the instance by deleting it.
89-
return false, inPlaceHelper.rebuildSourcePVCsAndRecreateInstance(reqCtx, cli, opsRes.OpsRequest, progressDetail)
83+
// no-backup rebuild has no data source to restore. Recreate the source
84+
// PVC from the current workload template instead of building helper
85+
// tmp PVCs/PVs that cannot carry backup data.
86+
return false, inPlaceHelper.rebuildSourcePVCsByDynamicProvision(reqCtx, cli, opsRes.OpsRequest, progressDetail)
9087
}
9188

92-
// 3. waiting for new instance is available.
89+
// waiting for new instance is available.
9390
return instanceIsAvailable(inPlaceHelper.synthesizedComp, inPlaceHelper.targetPod, opsRes.OpsRequest.Annotations[ignoreRoleCheckAnnotationKey])
9491
}
9592

@@ -439,6 +436,35 @@ func (inPlaceHelper *inplaceRebuildHelper) rebuildSourcePVCsAndRecreateInstance(
439436
return nil
440437
}
441438

439+
func (inPlaceHelper *inplaceRebuildHelper) rebuildSourcePVCsByDynamicProvision(reqCtx intctrlutil.RequestCtx,
440+
cli client.Client,
441+
opsRequest *opsv1alpha1.OpsRequest,
442+
progressDetail *opsv1alpha1.ProgressStatusDetail) error {
443+
itsName := constant.GenerateWorkloadNamePattern(inPlaceHelper.synthesizedComp.ClusterName, inPlaceHelper.synthesizedComp.Name)
444+
if err := inPlaceHelper.setInstanceNodeSelectorForRebuild(reqCtx, cli, itsName); err != nil {
445+
return err
446+
}
447+
for sourcePVCName, sourcePVCTemplate := range inPlaceHelper.pvcMap {
448+
sourcePVC, err := inPlaceHelper.getSourcePVC(reqCtx, cli, sourcePVCName, sourcePVCTemplate.Namespace)
449+
if err != nil {
450+
if apierrors.IsNotFound(err) {
451+
continue
452+
}
453+
return err
454+
}
455+
if sourcePVC.DeletionTimestamp == nil {
456+
if err := intctrlutil.BackgroundDeleteObject(cli, reqCtx.Ctx, sourcePVC); err != nil {
457+
return err
458+
}
459+
}
460+
}
461+
if err := inPlaceHelper.deleteTargetPodForRebuild(reqCtx, cli, opsRequest); err != nil {
462+
return err
463+
}
464+
progressDetail.Message = waitingForInstanceReadyMessage
465+
return nil
466+
}
467+
442468
func (inPlaceHelper *inplaceRebuildHelper) deleteTargetPodForRebuild(reqCtx intctrlutil.RequestCtx,
443469
cli client.Client,
444470
opsRequest *opsv1alpha1.OpsRequest) error {
@@ -699,7 +725,8 @@ func getPVCMapAndVolumes(opsRes *OpsResource,
699725
synthesizedComp *component.SynthesizedComponent,
700726
targetPod *corev1.Pod,
701727
rebuildPrefix string,
702-
index int) (map[string]*corev1.PersistentVolumeClaim, []corev1.Volume, []corev1.VolumeMount, error) {
728+
index int,
729+
noBackup bool) (map[string]*corev1.PersistentVolumeClaim, []corev1.Volume, []corev1.VolumeMount, error) {
703730
var (
704731
volumes []corev1.Volume
705732
volumeMounts []corev1.VolumeMount
@@ -727,6 +754,16 @@ func getPVCMapAndVolumes(opsRes *OpsResource,
727754
Spec: vct.Spec,
728755
}, workloadName, targetPod.Name)
729756
}
757+
if noBackup {
758+
pvcMap[sourcePVCName] = &corev1.PersistentVolumeClaim{
759+
ObjectMeta: metav1.ObjectMeta{
760+
Name: sourcePVCName,
761+
Namespace: targetPod.Namespace,
762+
},
763+
Spec: vct.Spec,
764+
}
765+
continue
766+
}
730767
pvcLabels := getWellKnownLabels(synthesizedComp)
731768
tmpPVC := &corev1.PersistentVolumeClaim{
732769
ObjectMeta: metav1.ObjectMeta{

‎pkg/operations/rebuild_instance_test.go‎

Lines changed: 81 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,79 @@ var _ = Describe("OpsUtil functions", func() {
361361
}
362362
}
363363

364+
sourcePVCsShouldDynamicReprovision := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource) {
365+
sourcePVCTemplateList := &corev1.PersistentVolumeClaimList{}
366+
Expect(k8sClient.List(ctx, sourcePVCTemplateList,
367+
client.MatchingLabels{constant.KBAppComponentLabelKey: defaultCompName},
368+
client.InNamespace(opsRes.OpsRequest.Namespace))).Should(Succeed())
369+
sourcePVCTemplates := map[string]*corev1.PersistentVolumeClaim{}
370+
targetSourcePVCs := map[string]bool{}
371+
for _, ins := range opsRes.OpsRequest.Spec.RebuildFrom[0].Instances {
372+
targetSourcePVCs[fmt.Sprintf("%s-%s", testapps.DataVolumeName, ins.Name)] = true
373+
}
374+
for i := range sourcePVCTemplateList.Items {
375+
pvc := sourcePVCTemplateList.Items[i].DeepCopy()
376+
if _, ok := pvc.Annotations[rebuildFromAnnotation]; ok {
377+
continue
378+
}
379+
if !targetSourcePVCs[pvc.Name] {
380+
continue
381+
}
382+
Expect(testapps.ChangeObj(&testCtx, pvc, func(p *corev1.PersistentVolumeClaim) {
383+
p.Finalizers = append(p.Finalizers, "kubernetes.io/pvc-protection")
384+
})).Should(Succeed())
385+
Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pvc), pvc)).Should(Succeed())
386+
sourcePVCTemplates[pvc.Name] = pvc
387+
}
388+
initialSourcePVCNames := make([]string, 0, len(sourcePVCTemplates))
389+
for sourcePVCName := range sourcePVCTemplates {
390+
initialSourcePVCNames = append(initialSourcePVCNames, sourcePVCName)
391+
}
392+
slices.Sort(initialSourcePVCNames)
393+
preDeletingSourcePVCName := initialSourcePVCNames[0]
394+
Expect(k8sClient.Delete(ctx, sourcePVCTemplates[preDeletingSourcePVCName])).Should(Succeed())
395+
396+
_, err := GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes)
397+
Expect(err).Should(Succeed())
398+
By("expect source PVC deletion to be requested before deleting the old target pod")
399+
for sourcePVCName := range sourcePVCTemplates {
400+
Eventually(func(g Gomega) {
401+
pvc := &corev1.PersistentVolumeClaim{}
402+
err := k8sClient.Get(ctx, client.ObjectKey{Name: sourcePVCName, Namespace: opsRes.OpsRequest.Namespace}, pvc)
403+
if apierrors.IsNotFound(err) {
404+
return
405+
}
406+
g.Expect(err).Should(Succeed())
407+
g.Expect(pvc.DeletionTimestamp).ShouldNot(BeNil())
408+
g.Expect(pvc.Finalizers).Should(ContainElement("kubernetes.io/pvc-protection"))
409+
}).Should(Succeed())
410+
}
411+
for _, ins := range opsRes.OpsRequest.Spec.RebuildFrom[0].Instances {
412+
Eventually(func(g Gomega) {
413+
pod := &corev1.Pod{}
414+
err := k8sClient.Get(ctx, client.ObjectKey{Name: ins.Name, Namespace: opsRes.OpsRequest.Namespace}, pod)
415+
g.Expect(apierrors.IsNotFound(err)).Should(BeTrue())
416+
}).Should(Succeed())
417+
}
418+
419+
By("expect old source PVCs to keep PVC protection until the workload releases them")
420+
for sourcePVCName := range sourcePVCTemplates {
421+
Eventually(func(g Gomega) {
422+
pvc := &corev1.PersistentVolumeClaim{}
423+
g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: sourcePVCName, Namespace: opsRes.OpsRequest.Namespace}, pvc)).Should(Succeed())
424+
g.Expect(pvc.DeletionTimestamp).ShouldNot(BeNil())
425+
g.Expect(pvc.Finalizers).Should(ContainElement("kubernetes.io/pvc-protection"))
426+
}).Should(Succeed())
427+
testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeClaimSignature, true,
428+
client.InNamespace(opsRes.OpsRequest.Namespace), client.MatchingFields{"metadata.name": sourcePVCName})
429+
}
430+
431+
By("expect rebuild to wait for the rebuilt instance")
432+
for _, detail := range opsRes.OpsRequest.Status.Components[defaultCompName].ProgressDetails {
433+
Expect(detail.Message).Should(Equal(waitingForInstanceReadyMessage))
434+
}
435+
}
436+
364437
waitForInstanceToAvailable := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource, ignoreRoleCheck bool) {
365438
By("waiting for the rebuild instance to ready")
366439
Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(opsRes.OpsRequest), func(g Gomega, ops *opsv1alpha1.OpsRequest) {
@@ -389,16 +462,18 @@ var _ = Describe("OpsUtil functions", func() {
389462
its := testapps.MockInstanceSetComponent(&testCtx, clusterName, defaultCompName)
390463
opsRes.OpsRequest.Status.Phase = opsv1alpha1.OpsRunningPhase
391464
reqCtx := intctrlutil.RequestCtx{Ctx: testCtx.Ctx}
392-
393-
By("expect for the tmp pods and pvcs are created")
394-
_, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes)
395465
matchingLabels := client.MatchingLabels{
396466
constant.OpsRequestNameLabelKey: opsRes.OpsRequest.Name,
397467
constant.OpsRequestNamespaceLabelKey: opsRes.OpsRequest.Namespace,
398468
}
469+
470+
By("expect to dynamically reprovision the source pvcs without helper tmp resources.")
471+
sourcePVCsShouldDynamicReprovision(reqCtx, opsRes)
472+
473+
By("expect no-backup rebuild to skip tmp pod/pvc creation")
399474
podList := &corev1.PodList{}
400475
Expect(k8sClient.List(ctx, podList, matchingLabels, client.InNamespace(opsRes.OpsRequest.Namespace))).Should(Succeed())
401-
Expect(podList.Items).Should(HaveLen(rebuildInstanceCount))
476+
Expect(podList.Items).Should(BeEmpty())
402477
pvcList := &corev1.PersistentVolumeClaimList{}
403478
Expect(k8sClient.List(ctx, pvcList, client.MatchingLabels{constant.KBAppComponentLabelKey: defaultCompName}, client.InNamespace(opsRes.OpsRequest.Namespace))).Should(Succeed())
404479
tmpPVCCount := 0
@@ -407,18 +482,7 @@ var _ = Describe("OpsUtil functions", func() {
407482
tmpPVCCount += 1
408483
}
409484
}
410-
Expect(tmpPVCCount).Should(Equal(rebuildInstanceCount))
411-
412-
By("fake the rebuilding pod to be Completed and fake pvs are created.")
413-
for i := range podList.Items {
414-
pod := &podList.Items[i]
415-
Expect(testapps.ChangeObjStatus(&testCtx, pod, func() {
416-
pod.Status.Phase = corev1.PodSucceeded
417-
})).Should(Succeed())
418-
}
419-
420-
By("expect to create the source pvcs and the pvs have rebind them.")
421-
sourcePVCsShouldRebindPVs(reqCtx, opsRes, pvcList)
485+
Expect(tmpPVCCount).Should(BeZero())
422486

423487
By("expect the opsRequest to succeed")
424488
waitForInstanceToAvailable(reqCtx, opsRes, false)
@@ -427,7 +491,7 @@ var _ = Describe("OpsUtil functions", func() {
427491
g.Expect(ops.Status.Phase).Should(Equal(opsv1alpha1.OpsSucceedPhase))
428492
}))
429493

430-
By("expect to clean up the tmp pods")
494+
By("expect no tmp pods were left behind")
431495
_, err := GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes)
432496
Expect(err).NotTo(HaveOccurred())
433497
Eventually(testapps.List(&testCtx, generics.PodSignature, matchingLabels, client.InNamespace(opsRes.OpsRequest.Namespace))).Should(HaveLen(0))

0 commit comments

Comments
 (0)