juicefs: support extra options in dataload (#4760)

* juicefs: support extra options in dataload

Signed-off-by: zwwhdls <zww@hdls.me>

* docs: add docs for options in dataload

Signed-off-by: zwwhdls <zww@hdls.me>

* fix unittest

Signed-off-by: zwwhdls <zww@hdls.me>

---------

Signed-off-by: zwwhdls <zww@hdls.me>

Author: zwwhdls

charts/fluid-dataloader/juicefs/templates/configmap.yaml

@@ -38,7 +38,7 @@ data:
           local pod="${podNames[i]}"
 
           echo -e "juicefs warmup on $pod $targetPath starts"
-          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/local/bin/juicefs warmup $targetPath
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/local/bin/juicefs warmup $targetPath $OPTION
           echo -e "juicefs warmup on $pod $targetPath ends"
         done
         fi
@@ -47,7 +47,7 @@ data:
         then
           echo -e "juicefs warmup $targetPath starts"
           local pod="${podNames[0]}"
-          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/bin/juicefs warmup $targetPath
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/bin/juicefs warmup $targetPath $OPTION
           echo -e "juicefs warmup $targetPath ends"
         fi
     }

charts/fluid-dataloader/juicefs/templates/cronjob.yaml

@@ -111,6 +111,10 @@ spec:
                 - name: TIMEOUT
                   value: {{ $val | quote }}
                 {{- end }}
+                {{- if eq $key "option" }}
+                - name: OPTION
+                  value: {{ $val | quote }}
+                {{- end }}
                 {{- if eq $key "edition" }}
                 - name: EDITION
                   value: {{ $val | quote }}

charts/fluid-dataloader/juicefs/templates/job.yaml

@@ -124,6 +124,10 @@ spec:
             - name: TIMEOUT
               value: {{ $val | quote }}
             {{- end }}
+            {{- if eq $key "option" }}
+            - name: OPTION
+              value: {{ $val | quote }}
+            {{- end }}
             {{- if eq $key "edition" }}
             - name: EDITION
               value: {{ $val | quote }}

docs/zh/samples/juicefs/juicefs_data_accelerate.md

@@ -21,9 +21,12 @@ spec:
     namespace: default
   target:
     - path: /dir1
+  options:
+    threads: "50"
 ```
 
 其中：
 - `spec.dataset.name`：指定要加速数据访问的 Dataset 名称；
 - `spec.dataset.namespce`：指定要加速数据访问的 Dataset 的 namespace；
 - `spec.target.path`：指定要加速数据访问的数据路径，可以是目录或文件；target 是列表，可以填多个 path。
+- `spec.options`：指定缓存加速的参数，可用参数可参考 [JuiceFS 的缓存加速文档](https://juicefs.com/docs/zh/community/command_reference#warmup)，注意 key 和 value 均为字符串类型。

pkg/ddc/juicefs/data_load.go

@@ -179,12 +179,6 @@ func (j *JuiceFSEngine) genDataLoadValue(image string, cacheinfo map[string]stri
 	dataloadInfo.TargetPaths = targetPaths
 
 	options := map[string]string{}
-	// resolve spec options
-	if dataload.Spec.Options != nil {
-		for key, value := range dataload.Spec.Options {
-			options[key] = value
-		}
-	}
 
 	for key, value := range cacheinfo {
 		options[key] = value
@@ -201,9 +195,21 @@ func (j *JuiceFSEngine) genDataLoadValue(image string, cacheinfo map[string]stri
 	}
 	options["edition"] = cacheinfo[Edition]
 	options["runtimeName"] = j.name
-	if _, ok := options["timeout"]; !ok {
-		options["timeout"] = DefaultDataLoadTimeout
+	timeout := dataload.Spec.Options["timeout"]
+	delete(dataload.Spec.Options, "timeout")
+	if timeout == "" {
+		timeout = DefaultDataLoadTimeout
+	}
+	warmupOtions := []string{}
+	for k, v := range dataload.Spec.Options {
+		if v != "" {
+			warmupOtions = append(warmupOtions, fmt.Sprintf("--%s=%s", k, v))
+		} else {
+			warmupOtions = append(warmupOtions, fmt.Sprintf("--%s", k))
+		}
 	}
+	options["option"] = strings.Join(warmupOtions, " ")
+	options["timeout"] = timeout
 
 	dataloadInfo.Options = options
 

pkg/ddc/juicefs/data_load_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package juicefs
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -25,14 +26,15 @@ import (
 
 	"github.com/fluid-cloudnative/fluid/pkg/common"
 
-	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
-	cdataload "github.com/fluid-cloudnative/fluid/pkg/dataload"
-	cruntime "github.com/fluid-cloudnative/fluid/pkg/runtime"
-	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+
+	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
+	cdataload "github.com/fluid-cloudnative/fluid/pkg/dataload"
+	cruntime "github.com/fluid-cloudnative/fluid/pkg/runtime"
+	"github.com/fluid-cloudnative/fluid/pkg/utils/fake"
 )
 
 var valuesConfigMapData = `
@@ -561,7 +563,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					ImagePullSecrets: []v1.LocalObjectReference{},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -728,7 +730,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -829,7 +831,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -938,7 +940,7 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 					},
 					Options: map[string]string{
 						// dataload spec options
-						"dl-opts-k-1": "dl-opts-v-1",
+						"option": "--dl-opts-k-1=dl-opts-v-1",
 						// cache info
 						Edition:         CommunityEdition,
 						"cache-info-k1": "cache-info-v1",
@@ -961,7 +963,9 @@ func TestJuiceFSEngine_genDataLoadValue(t *testing.T) {
 		}
 		got, _ := engine.genDataLoadValue(item.image, item.cacheInfo, item.pods, item.targetDataset, item.dataload)
 		if !reflect.DeepEqual(got, item.want) {
-			t.Errorf("case %s, got %v,want:%v", k, got, item.want)
+			gotJson, _ := json.Marshal(got)
+			wantJson, _ := json.Marshal(item.want)
+			t.Errorf("case %s, got %v\nwant:%v", k, string(gotJson), string(wantJson))
 		}
 	}
 }