Skip to content

Commit cffa741

Browse files
staerionstaerion
committed
allow to set resource requests and limits for deamonset through operator
1 parent 31c02b2 commit cffa741

File tree

6 files changed

+309
-113
lines changed

6 files changed

+309
-113
lines changed

‎cli/k8s_client/types.go‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"k8s.io/apimachinery/pkg/types"
1717
"k8s.io/apimachinery/pkg/version"
1818

19+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
1920
versionutils "github.com/netapp/trident/utils/version"
2021
)
2122

@@ -168,27 +169,29 @@ type DeploymentYAMLArguments struct {
168169
}
169170

170171
type DaemonsetYAMLArguments struct {
171-
DaemonsetName string `json:"daemonsetName"`
172-
TridentImage string `json:"tridentImage"`
173-
ImageRegistry string `json:"imageRegistry"`
174-
KubeletDir string `json:"kubeletDir"`
175-
LogFormat string `json:"logFormat"`
176-
LogLevel string `json:"logLevel"`
177-
LogWorkflows string `json:"logWorkflows"`
178-
LogLayers string `json:"logLayers"`
179-
ProbePort string `json:"probePort"`
180-
ImagePullSecrets []string `json:"imagePullSecrets"`
181-
Labels map[string]string `json:"labels"`
182-
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
183-
EnableForceDetach bool `json:"enableForceDetach"`
184-
DisableAuditLog bool `json:"disableAuditLog"`
185-
Debug bool `json:"debug"`
186-
Version *versionutils.Version `json:"version"`
187-
HTTPRequestTimeout string `json:"httpRequestTimeout"`
188-
NodeSelector map[string]string `json:"nodeSelector"`
189-
Tolerations []map[string]string `json:"tolerations"`
190-
ServiceAccountName string `json:"serviceAccountName"`
191-
ImagePullPolicy string `json:"imagePullPolicy"`
172+
DaemonsetName string `json:"daemonsetName"`
173+
TridentImage string `json:"tridentImage"`
174+
ImageRegistry string `json:"imageRegistry"`
175+
KubeletDir string `json:"kubeletDir"`
176+
LogFormat string `json:"logFormat"`
177+
LogLevel string `json:"logLevel"`
178+
LogWorkflows string `json:"logWorkflows"`
179+
LogLayers string `json:"logLayers"`
180+
ProbePort string `json:"probePort"`
181+
ImagePullSecrets []string `json:"imagePullSecrets"`
182+
Labels map[string]string `json:"labels"`
183+
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
184+
EnableForceDetach bool `json:"enableForceDetach"`
185+
DisableAuditLog bool `json:"disableAuditLog"`
186+
Debug bool `json:"debug"`
187+
Version *versionutils.Version `json:"version"`
188+
HTTPRequestTimeout string `json:"httpRequestTimeout"`
189+
NodeSelector map[string]string `json:"nodeSelector"`
190+
Tolerations []map[string]string `json:"tolerations"`
191+
TridentResourceSpec netappv1.ResourceSpec `json:"tridentResourceSpec"`
192+
RegistrarResourceSpec netappv1.ResourceSpec `json:"registrarSpec"`
193+
ServiceAccountName string `json:"serviceAccountName"`
194+
ImagePullPolicy string `json:"imagePullPolicy"`
192195
}
193196

194197
type TridentVersionPodYAMLArguments struct {

‎cli/k8s_client/yaml_factory.go‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ import (
99

1010
commonconfig "github.com/netapp/trident/config"
1111
. "github.com/netapp/trident/logging"
12+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
1213
"github.com/netapp/trident/utils"
14+
v1 "k8s.io/api/core/v1"
15+
"k8s.io/apimachinery/pkg/api/resource"
16+
k8syaml "sigs.k8s.io/yaml"
1317
)
1418

1519
const (
@@ -726,6 +730,25 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
726730
}
727731
}
728732

733+
defaultRegistrarResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
734+
Limits: v1.ResourceList{
735+
v1.ResourceMemory: resource.MustParse("100Mi"),
736+
},
737+
Requests: v1.ResourceList{
738+
v1.ResourceCPU: resource.MustParse("10m"),
739+
v1.ResourceMemory: resource.MustParse("40Mi"),
740+
},
741+
}}
742+
defaultTridentResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
743+
Limits: v1.ResourceList{
744+
v1.ResourceMemory: resource.MustParse("400Mi"),
745+
},
746+
Requests: v1.ResourceList{
747+
v1.ResourceCPU: resource.MustParse("10m"),
748+
v1.ResourceMemory: resource.MustParse("20Mi"),
749+
},
750+
}}
751+
729752
kubeletDir := strings.TrimRight(args.KubeletDir, "/")
730753
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
731754
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
@@ -745,6 +768,8 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
745768
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
746769
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
747770
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
771+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim("DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResources))
772+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim("DsTrident", args.TridentResourceSpec, defaultTridentResources))
748773
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
749774
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
750775
// Log before secrets are inserted into YAML.
@@ -799,6 +824,13 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
799824
}
800825
}
801826

827+
defaultRegistrarResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
828+
Requests: v1.ResourceList{},
829+
}}
830+
defaultTridentResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
831+
Requests: v1.ResourceList{},
832+
}}
833+
802834
kubeletDir := strings.TrimRight(args.KubeletDir, "/")
803835
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
804836
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
@@ -819,6 +851,8 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
819851
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
820852
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
821853
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
854+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim("DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResources))
855+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim("DsTrident", args.TridentResourceSpec, defaultTridentResources))
822856
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
823857
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
824858

@@ -907,6 +941,7 @@ spec:
907941
failureThreshold: 5
908942
initialDelaySeconds: 10
909943
periodSeconds: 10
944+
{TRIDENT_RESOURCE_CLAIMS}
910945
env:
911946
- name: KUBE_NODE_NAME
912947
valueFrom:
@@ -948,6 +983,7 @@ spec:
948983
- "--v={SIDECAR_LOG_LEVEL}"
949984
- "--csi-address=$(ADDRESS)"
950985
- "--kubelet-registration-path=$(REGISTRATION_PATH)"
986+
{REGISTRAR_RESOURCE_CLAIMS}
951987
env:
952988
- name: ADDRESS
953989
value: /plugin/csi.sock
@@ -1137,12 +1173,7 @@ spec:
11371173
mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1
11381174
- name: csi-proxy-smb-pipe-v1beta1
11391175
mountPath: \\.\pipe\csi-proxy-smb-v1beta1
1140-
resources:
1141-
limits:
1142-
memory: 400Mi
1143-
requests:
1144-
cpu: 10m
1145-
memory: 20Mi
1176+
{TRIDENT_RESOURCE_CLAIMS}
11461177
- name: node-driver-registrar
11471178
image: {CSI_SIDECAR_REGISTRY}/csi-node-driver-registrar:v2.8.0
11481179
imagePullPolicy: {IMAGE_PULL_POLICY}
@@ -1174,12 +1205,7 @@ spec:
11741205
mountPath: C:\csi
11751206
- name: registration-dir
11761207
mountPath: C:\registration
1177-
resources:
1178-
limits:
1179-
memory: 200Mi
1180-
requests:
1181-
cpu: 10m
1182-
memory: 20Mi
1208+
{REGISTRAR_RESOURCE_CLAIMS}
11831209
- name: liveness-probe
11841210
volumeMounts:
11851211
- mountPath: C:\csi
@@ -1193,12 +1219,6 @@ spec:
11931219
env:
11941220
- name: CSI_ENDPOINT
11951221
value: unix:///csi/csi.sock
1196-
resources:
1197-
limits:
1198-
memory: 100Mi
1199-
requests:
1200-
cpu: 10m
1201-
memory: 40Mi
12021222
affinity:
12031223
nodeAffinity:
12041224
requiredDuringSchedulingIgnoredDuringExecution:
@@ -2509,6 +2529,25 @@ func constructNodeSelector(nodeLabels map[string]string) string {
25092529
return nodeSelector
25102530
}
25112531

2532+
func constructResourceClaim(containerName string, resourceClaim netappv1.ResourceSpec, defaultResourceClaim netappv1.ResourceSpec) string {
2533+
// defaultResult, _ := k8syaml.Marshal(defaultResourceClaim)
2534+
2535+
// err := k8syaml.Unmarshal(defaultResult, &resourceClaim)
2536+
// result, err := k8syaml.Marshal(resourceClaim)
2537+
2538+
userData, _ := k8syaml.Marshal(resourceClaim)
2539+
2540+
err := k8syaml.Unmarshal(userData, &defaultResourceClaim)
2541+
mergedData, err := k8syaml.Marshal(defaultResourceClaim)
2542+
2543+
if err != nil {
2544+
Log().WithField("container", containerName).Errorf("Cannot marshal resource claims: %s", err)
2545+
return ""
2546+
}
2547+
2548+
return string(mergedData)
2549+
}
2550+
25122551
func constructTolerations(tolerations []map[string]string) string {
25132552
var tolerationsString string
25142553

‎cli/k8s_client/yaml_factory_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ import (
1515
pspv1beta1 "k8s.io/api/policy/v1beta1"
1616
csiv1 "k8s.io/api/storage/v1"
1717
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
18+
"k8s.io/apimachinery/pkg/api/resource"
1819
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1920

2021
"github.com/netapp/trident/config"
22+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
2123
versionutils "github.com/netapp/trident/utils/version"
2224
)
2325

@@ -652,6 +654,43 @@ func TestGetCSIDaemonSetYAMLLinux_NodeSelectors(t *testing.T) {
652654
fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))
653655
}
654656

657+
func TestGetCSIDaemonSetYAMLLinux_ResourceClaims(t *testing.T) {
658+
daemonsetArgs := &DaemonsetYAMLArguments{
659+
RegistrarResourceSpec: netappv1.ResourceSpec{},
660+
TridentResourceSpec: netappv1.ResourceSpec{
661+
Resources: v1.ResourceRequirements{
662+
Limits: v1.ResourceList{
663+
v1.ResourceMemory: resource.MustParse("125Mi"),
664+
},
665+
Requests: v1.ResourceList{
666+
v1.ResourceCPU: resource.MustParse("50m"),
667+
v1.ResourceMemory: resource.MustParse("50Mi"),
668+
},
669+
},
670+
},
671+
}
672+
expectedRegistrarResourceClaimString := "resources: {}\n"
673+
expectedTridentResourceClaimString := `
674+
resources:
675+
limits:
676+
memory: 125Mi
677+
requests:
678+
cpu: 50m
679+
memory: 50Mi
680+
`
681+
682+
yamlData := GetCSIDaemonSetYAMLLinux(daemonsetArgs)
683+
_, err := yaml.YAMLToJSON([]byte(yamlData))
684+
if err != nil {
685+
t.Fatalf("expected valid YAML, got %s", yamlData)
686+
}
687+
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
688+
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))
689+
690+
assert.Contains(t, yamlData, expectedTridentResourceClaimString,
691+
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
692+
}
693+
655694
func TestGetCSIDaemonSetYAMLLinux_Tolerations(t *testing.T) {
656695
daemonsetArgs := &DaemonsetYAMLArguments{
657696
Tolerations: []map[string]string{
@@ -800,6 +839,56 @@ func TestGetCSIDaemonSetYAMLWindowsImagePullPolicy(t *testing.T) {
800839
}
801840
}
802841

842+
func TestGetCSIDaemonSetYAMWindows_ResourceClaims(t *testing.T) {
843+
versions := []string{"1.26.0"}
844+
registrarResourceSpec := netappv1.ResourceSpec{}
845+
tridentResourceSpec := netappv1.ResourceSpec{
846+
Resources: v1.ResourceRequirements{
847+
Limits: v1.ResourceList{
848+
v1.ResourceMemory: resource.MustParse("125Mi"),
849+
},
850+
Requests: v1.ResourceList{
851+
v1.ResourceCPU: resource.MustParse("50m"),
852+
},
853+
},
854+
}
855+
856+
expectedRegistrarResourceClaimString := `
857+
resources:
858+
limits:
859+
memory: 100Mi
860+
requests:
861+
cpu: 10m
862+
memory: 40Mi
863+
`
864+
expectedTridentResourceClaimString := `
865+
resources:
866+
limits:
867+
memory: 125Mi
868+
requests:
869+
cpu: 50m
870+
memory: 20Mi
871+
`
872+
for _, versionString := range versions {
873+
version := versionutils.MustParseSemantic(versionString)
874+
daemonsetArgs := &DaemonsetYAMLArguments{
875+
Version: version,
876+
RegistrarResourceSpec: registrarResourceSpec,
877+
TridentResourceSpec: tridentResourceSpec,
878+
}
879+
yamlData := GetCSIDaemonSetYAMLWindows(daemonsetArgs)
880+
_, err := yaml.YAMLToJSON([]byte(yamlData))
881+
if err != nil {
882+
t.Fatalf("expected valid YAML, got %s", yamlData)
883+
}
884+
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
885+
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))
886+
887+
assert.Contains(t, yamlData, expectedTridentResourceClaimString,
888+
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
889+
}
890+
}
891+
803892
func TestConstructNodeSelector(t *testing.T) {
804893
nodeSelMap := map[string]string{"node-label-name": "master"}
805894

@@ -809,6 +898,54 @@ func TestConstructNodeSelector(t *testing.T) {
809898
assert.Equal(t, expectedNodeSelString, result)
810899
}
811900

901+
func TestConstructResourceClaim(t *testing.T) {
902+
defaultResourceClaim := netappv1.ResourceSpec{
903+
Resources: v1.ResourceRequirements{},
904+
}
905+
resourceClaim := netappv1.ResourceSpec{
906+
Resources: v1.ResourceRequirements{
907+
Limits: v1.ResourceList{
908+
v1.ResourceCPU: resource.MustParse("100m"),
909+
v1.ResourceMemory: resource.MustParse("50Mi"),
910+
},
911+
Requests: v1.ResourceList{
912+
v1.ResourceCPU: resource.MustParse("20m"),
913+
v1.ResourceMemory: resource.MustParse("20Mi"),
914+
},
915+
},
916+
}
917+
918+
expectedResourceClaimString := `resources:
919+
limits:
920+
cpu: 100m
921+
memory: 50Mi
922+
requests:
923+
cpu: 20m
924+
memory: 20Mi
925+
`
926+
927+
result := constructResourceClaim("test", resourceClaim, defaultResourceClaim)
928+
assert.Equal(t, expectedResourceClaimString, result)
929+
930+
resourceClaim = netappv1.ResourceSpec{
931+
Resources: v1.ResourceRequirements{
932+
Limits: v1.ResourceList{
933+
v1.ResourceCPU: resource.MustParse("100m"),
934+
},
935+
},
936+
}
937+
938+
expectedResourceClaimString = "resources:\n limits:\n cpu: 100m\n"
939+
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
940+
assert.Equal(t, expectedResourceClaimString, result)
941+
942+
resourceClaim = netappv1.ResourceSpec{}
943+
944+
expectedResourceClaimString = "resources: {}\n"
945+
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
946+
assert.Equal(t, expectedResourceClaimString, result)
947+
}
948+
812949
func TestGetNamespaceYAML(t *testing.T) {
813950
expected := v1.Namespace{
814951
TypeMeta: metav1.TypeMeta{

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ require (
5555
k8s.io/apimachinery v0.27.3 // github.com/kubernetes/apimachinery
5656
k8s.io/client-go v0.27.3 // github.com/kubernetes/client-go
5757
k8s.io/mount-utils v0.27.3 // github.com/kubernetes/mount-utils
58+
sigs.k8s.io/yaml v1.3.0
5859
)
5960

6061
require (
@@ -132,5 +133,4 @@ require (
132133
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
133134
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
134135
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
135-
sigs.k8s.io/yaml v1.3.0 // indirect
136136
)

0 commit comments

Comments
 (0)