Skip to content

Commit 82e490e

Browse files
xliuqqxliuqq
authored
Extend CacheRuntime phase 3: add dataload/dataprocess support (#5798)
* add dataload/dataprocess for curvine Signed-off-by: xliuqq <xlzq1992@gmail.com> add imagePullPolicy Signed-off-by: xliuqq <xlzq1992@gmail.com> add test Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix coplit suggestions Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix symbol link Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix review Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix cache engine CheckRuntimeReady for no master architecture Signed-off-by: xliuqq <xlzq1992@gmail.com> --------- Signed-off-by: xliuqq <xlzq1992@gmail.com>
1 parent 6df6421 commit 82e490e

33 files changed

Lines changed: 1057 additions & 53 deletions

‎api/v1alpha1/cacheruntimeclass_types.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,26 @@ type CacheRuntimeClass struct {
179179
// ExtraResources specifies additional resources (e.g., ConfigMaps) used by the CacheRuntime components
180180
// +optional
181181
ExtraResources RuntimeExtraResources `json:"extraResources,omitempty"`
182+
183+
// DataOperationSpecs specifies the data operation spec
184+
// +optional
185+
DataOperationSpecs []DataOperationSpec `json:"dataOperationSpecs,omitempty"`
186+
}
187+
188+
type DataOperationSpec struct {
189+
// Name is the data operation name like DataLoad, DataBackup, DataMigrate etc.
190+
// +kubebuilder:validation:Enum=DataLoad;DataBackup;DataMigrate;DataProcess
191+
Name string `json:"name"`
192+
193+
// Image the image for data operation, if not existed, use the runtime/runtimeclass defined worker image.
194+
// +optional
195+
Image string `json:"image,omitempty"`
196+
197+
// Command for data operation Pod container
198+
Command []string `json:"command,omitempty"`
199+
200+
// Args for data operation Pod container
201+
Args []string `json:"args,omitempty"`
182202
}
183203

184204
// CacheRuntimeClassList contains a list of CacheRuntimeClass

‎api/v1alpha1/openapi_generated.go‎

Lines changed: 74 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/fluid-dataloader/cache/Chart.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: v2
2+
name: fluid-dataloader
3+
description: A Helm chart for Fluid to prefetch data
4+
5+
# A chart can be either an 'application' or a 'library' chart.
6+
#
7+
# Application charts are a collection of templates that can be packaged into versioned archives
8+
# to be deployed.
9+
#
10+
# Library charts provide useful utilities or functions for the chart developer. They're included as
11+
# a dependency of application charts to inject those utilities and functions into the rendering
12+
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
13+
type: application
14+
15+
# This is the chart version. This version number should be incremented each time you make changes
16+
# to the chart and its templates, including the app version.
17+
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18+
version: 0.1.0
19+
20+
# This is the version number of the application being deployed. This version number should be
21+
# incremented each time you make changes to the application. Versions are not expected to
22+
# follow Semantic Versioning. They should reflect the version the application is using.
23+
appVersion: 0.3.0
24+
25+
dependencies:
26+
- name: library
27+
version: "0.2.0"
28+
repository: "file://../../library"

‎charts/fluid-dataloader/cache/charts/library‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../../../library

‎charts/fluid-dataloader/cache/templates/cronjob.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
{{- if eq (lower .Values.dataloader.policy) "cron" }}
2+
apiVersion: {{ ternary "batch/v1" "batch/v1beta1" (.Capabilities.APIVersions.Has "batch/v1/CronJob") }}
3+
kind: CronJob
4+
metadata:
5+
name: {{ printf "%s-job" .Release.Name }}
6+
labels:
7+
release: {{ .Release.Name }}
8+
role: dataload-cronjob
9+
app: cache
10+
targetDataset: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}
11+
dataload: {{ .Values.name }}
12+
fluid.io/jobPolicy: cron
13+
{{- include "library.fluid.labels" . | nindent 4 }}
14+
ownerReferences:
15+
{{- if .Values.owner.enabled }}
16+
- apiVersion: {{ .Values.owner.apiVersion }}
17+
blockOwnerDeletion: {{ .Values.owner.blockOwnerDeletion }}
18+
controller: {{ .Values.owner.controller }}
19+
kind: {{ .Values.owner.kind }}
20+
name: {{ .Values.owner.name }}
21+
uid: {{ .Values.owner.uid }}
22+
{{- end }}
23+
spec:
24+
schedule: "{{ .Values.dataloader.schedule }}"
25+
jobTemplate:
26+
spec:
27+
backoffLimit: {{ .Values.dataloader.backoffLimit | default "3" }}
28+
completions: 1
29+
parallelism: 1
30+
template:
31+
metadata:
32+
name: {{ printf "%s-loader" .Release.Name }}
33+
annotations:
34+
sidecar.istio.io/inject: "false"
35+
{{- if .Values.dataloader.annotations }}
36+
{{- range $key, $val := .Values.dataloader.annotations }}
37+
{{ $key | quote }}: {{ $val | quote }}
38+
{{- end }}
39+
{{- end }}
40+
labels:
41+
release: {{ .Release.Name }}
42+
role: dataload-pod
43+
app: cache
44+
cronjob: {{ printf "%s-job" .Release.Name }}
45+
targetDataset: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}
46+
{{- include "library.fluid.labels" . | nindent 12 }}
47+
{{- if .Values.dataloader.labels }}
48+
{{- range $key, $val := .Values.dataloader.labels }}
49+
{{ $key | quote }}: {{ $val | quote }}
50+
{{- end }}
51+
{{- end }}
52+
spec:
53+
{{- include "library.fluid.dataload.cronJobCommonTemplateSpec" . | nindent 10 }}
54+
containers:
55+
- name: dataloader
56+
image: {{ required "Dataloader image should be set" .Values.dataloader.image }}
57+
imagePullPolicy: IfNotPresent
58+
command:
59+
{{- range .Values.dataloader.command }}
60+
- {{ . | quote }}
61+
{{- end }}
62+
args:
63+
{{- range .Values.dataloader.args }}
64+
- {{ . | quote }}
65+
{{- end }}
66+
67+
{{- if .Values.dataloader.resources }}
68+
resources:
69+
{{- toYaml .Values.dataloader.resources | nindent 16}}
70+
{{- end }}
71+
{{- $targetPaths := "" }}
72+
{{- range .Values.dataloader.targetPaths }}
73+
{{- $targetPaths = cat $targetPaths (required "Path must be set" .path) ":" }}
74+
{{- end }}
75+
{{- $targetPaths = $targetPaths | nospace | trimSuffix ":" }}
76+
77+
{{- $pathReplicas := ""}}
78+
{{- range .Values.dataloader.targetPaths }}
79+
{{- $pathReplicas = cat $pathReplicas ( default 1 .replicas ) ":"}}
80+
{{- end }}
81+
{{- $pathReplicas = $pathReplicas | nospace | trimSuffix ":"}}
82+
env:
83+
- name: FLUID_DATALOAD_METADATA
84+
value: {{ default false .Values.dataloader.loadMetadata | quote }}
85+
- name: FLUID_DATALOAD_DATA_PATH
86+
value: {{ $targetPaths | quote }}
87+
- name: FLUID_DATALOAD_PATH_REPLICAS
88+
value: {{ $pathReplicas | quote }}
89+
{{- range .Values.dataloader.envs }}
90+
- name: {{ .name }}
91+
value: {{ .value | quote }}
92+
{{- end }}
93+
94+
{{- end }}

‎charts/fluid-dataloader/cache/templates/job.yaml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
{{- if or (eq (lower .Values.dataloader.policy) "") (eq (lower .Values.dataloader.policy) "once") }}
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: {{ printf "%s-job" .Release.Name }}
6+
labels:
7+
release: {{ .Release.Name }}
8+
role: dataload-job
9+
app: cache
10+
targetDataset: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}
11+
{{- include "library.fluid.labels" . | nindent 4 }}
12+
ownerReferences:
13+
{{- if .Values.owner.enabled }}
14+
- apiVersion: {{ .Values.owner.apiVersion }}
15+
blockOwnerDeletion: {{ .Values.owner.blockOwnerDeletion }}
16+
controller: {{ .Values.owner.controller }}
17+
kind: {{ .Values.owner.kind }}
18+
name: {{ .Values.owner.name }}
19+
uid: {{ .Values.owner.uid }}
20+
{{- end }}
21+
spec:
22+
backoffLimit: {{ .Values.dataloader.backoffLimit | default "3" }}
23+
completions: 1
24+
parallelism: 1
25+
template:
26+
metadata:
27+
name: {{ printf "%s-loader" .Release.Name }}
28+
annotations:
29+
sidecar.istio.io/inject: "false"
30+
{{- if .Values.dataloader.annotations }}
31+
{{- range $key, $val := .Values.dataloader.annotations }}
32+
{{ $key | quote }}: {{ $val | quote }}
33+
{{- end }}
34+
{{- end }}
35+
labels:
36+
release: {{ .Release.Name }}
37+
role: dataload-pod
38+
app: cache
39+
targetDataset: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}
40+
{{- include "library.fluid.labels" . | nindent 8 }}
41+
{{- if .Values.dataloader.labels }}
42+
{{- range $key, $val := .Values.dataloader.labels }}
43+
{{ $key | quote }}: {{ $val | quote }}
44+
{{- end }}
45+
{{- end }}
46+
spec:
47+
{{- if .Values.dataloader.schedulerName }}
48+
schedulerName: {{ .Values.dataloader.schedulerName }}
49+
{{- end }}
50+
{{- with .Values.dataloader.nodeSelector }}
51+
nodeSelector:
52+
{{- toYaml . | nindent 8 }}
53+
{{- end }}
54+
{{- with .Values.dataloader.affinity }}
55+
affinity:
56+
{{- toYaml . | nindent 8 }}
57+
{{- end }}
58+
{{- with .Values.dataloader.tolerations }}
59+
tolerations:
60+
{{- toYaml . | nindent 8 }}
61+
{{- end }}
62+
restartPolicy: Never
63+
{{- with .Values.dataloader.imagePullSecrets }}
64+
imagePullSecrets:
65+
{{- toYaml . | nindent 8 }}
66+
{{- end }}
67+
containers:
68+
- name: dataloader
69+
image: {{ required "Dataloader image should be set" .Values.dataloader.image }}
70+
imagePullPolicy: IfNotPresent
71+
command:
72+
{{- range .Values.dataloader.command }}
73+
- {{ . | quote }}
74+
{{- end }}
75+
args:
76+
{{- range .Values.dataloader.args }}
77+
- {{ . | quote }}
78+
{{- end }}
79+
80+
{{- if .Values.dataloader.resources }}
81+
resources:
82+
{{ toYaml .Values.dataloader.resources | nindent 12 }}
83+
{{- end }}
84+
{{- $targetPaths := "" }}
85+
{{- range .Values.dataloader.targetPaths }}
86+
{{- $targetPaths = cat $targetPaths (required "Path must be set" .path) ":" }}
87+
{{- end }}
88+
{{- $targetPaths = $targetPaths | nospace | trimSuffix ":" }}
89+
90+
{{- $pathReplicas := ""}}
91+
{{- range .Values.dataloader.targetPaths }}
92+
{{- $pathReplicas = cat $pathReplicas ( default 1 .replicas ) ":"}}
93+
{{- end }}
94+
{{- $pathReplicas = $pathReplicas | nospace | trimSuffix ":"}}
95+
env:
96+
- name: FLUID_DATALOAD_METADATA
97+
value: {{ default false .Values.dataloader.loadMetadata | quote }}
98+
- name: FLUID_DATALOAD_DATA_PATH
99+
value: {{ $targetPaths | quote }}
100+
- name: FLUID_DATALOAD_PATH_REPLICAS
101+
value: {{ $pathReplicas | quote }}
102+
{{- range .Values.dataloader.envs }}
103+
- name: {{ .name }}
104+
value: {{ .value | quote }}
105+
{{- end }}
106+
{{- end }}

0 commit comments

Comments
 (0)