fix(ddc): wrap ufsTotal status update with retry on conflict

pkg/ddc/alluxio/ufs_internal.go

@@ -19,12 +19,13 @@
 import (
 	"context"
 	"fmt"
-	v1 "k8s.io/api/core/v1"
 	"path/filepath"
 	"reflect"
 	"strings"
 	"time"
 
+	v1 "k8s.io/api/core/v1"
+
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	"github.com/fluid-cloudnative/fluid/pkg/common"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio/operations"
@@ -149,7 +150,7 @@
 	return fileUtils.FindUnmountedAlluxioPaths(alluxioPaths)
 }
 
 func (e *AlluxioEngine) processUpdatingUFS(ufsToUpdate *utils.UFSToUpdate) (updateReady bool, err error) {
 	dataset, err := utils.GetDataset(e.Client, e.name, e.namespace)
 	if err != nil {
 		return false, err
@@ -167,13 +168,24 @@
 
 	if updateReady {
 		// need to reset ufsTotal to Calculating so that SyncMetadata will work
-		datasetToUpdate := dataset.DeepCopy()
-		datasetToUpdate.Status.UfsTotal = metadataSyncNotDoneMsg
-		if !reflect.DeepEqual(dataset.Status, datasetToUpdate.Status) {
-			err = e.Client.Status().Update(context.TODO(), datasetToUpdate)
+		err = retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+			dataset, err := utils.GetDataset(e.Client, e.name, e.namespace)
 			if err != nil {
-				e.Log.Error(err, "fail to update ufsTotal of dataset to Calculating")
+				return
 			}
+			datasetToUpdate := dataset.DeepCopy()
+			datasetToUpdate.Status.UfsTotal = metadataSyncNotDoneMsg
+			if !reflect.DeepEqual(dataset.Status, datasetToUpdate.Status) {
+				err = e.Client.Status().Update(context.TODO(), datasetToUpdate)
+				if err != nil {
+					return
+				}
+			}
+			return
+		})
+		if err != nil {
+			e.Log.Error(err, "fail to update ufsTotal of dataset to Calculating")
+			return true, err
 		}
 
 		err = e.SyncMetadata()

pkg/ddc/goosefs/ufs_internal.go

@@ -28,6 +28,7 @@ import (
 	"github.com/fluid-cloudnative/fluid/pkg/utils/kubeclient"
 	securityutil "github.com/fluid-cloudnative/fluid/pkg/utils/security"
 	"github.com/pkg/errors"
+	"k8s.io/client-go/util/retry"
 )
 
 func (e *GooseFSEngine) usedStorageBytesInternal() (value int64, err error) {
@@ -235,13 +236,24 @@ func (e *GooseFSEngine) processUpdatingUFS(ufsToUpdate *utils.UFSToUpdate) (err
 		}
 	}
 	// need to reset ufsTotal to Calculating so that SyncMetadata will work
-	datasetToUpdate := dataset.DeepCopy()
-	datasetToUpdate.Status.UfsTotal = MetadataSyncNotDoneMsg
-	if !reflect.DeepEqual(dataset.Status, datasetToUpdate.Status) {
-		err = e.Client.Status().Update(context.TODO(), datasetToUpdate)
+	err = retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		dataset, err := utils.GetDataset(e.Client, e.name, e.namespace)
 		if err != nil {
-			e.Log.Error(err, "fail to update ufsTotal of dataset to Calculating")
+			return
 		}
+		datasetToUpdate := dataset.DeepCopy()
+		datasetToUpdate.Status.UfsTotal = MetadataSyncNotDoneMsg
+		if !reflect.DeepEqual(dataset.Status, datasetToUpdate.Status) {
+			err = e.Client.Status().Update(context.TODO(), datasetToUpdate)
+			if err != nil {
+				return
+			}
+		}
+		return
+	})
+	if err != nil {
+		e.Log.Error(err, "fail to update ufsTotal of dataset to Calculating")
+		return err
 	}
 
 	err = e.SyncMetadata()