Skip to content

Commit 3882803

Browse files
staerionstaerion
committed
allow to set resource requests and limits for deamonset through operator
1 parent 31c02b2 commit 3882803

File tree

6 files changed

+317
-113
lines changed

6 files changed

+317
-113
lines changed

‎cli/k8s_client/types.go‎

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"k8s.io/apimachinery/pkg/types"
1717
"k8s.io/apimachinery/pkg/version"
1818

19+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
1920
versionutils "github.com/netapp/trident/utils/version"
2021
)
2122

@@ -168,27 +169,29 @@ type DeploymentYAMLArguments struct {
168169
}
169170

170171
type DaemonsetYAMLArguments struct {
171-
DaemonsetName string `json:"daemonsetName"`
172-
TridentImage string `json:"tridentImage"`
173-
ImageRegistry string `json:"imageRegistry"`
174-
KubeletDir string `json:"kubeletDir"`
175-
LogFormat string `json:"logFormat"`
176-
LogLevel string `json:"logLevel"`
177-
LogWorkflows string `json:"logWorkflows"`
178-
LogLayers string `json:"logLayers"`
179-
ProbePort string `json:"probePort"`
180-
ImagePullSecrets []string `json:"imagePullSecrets"`
181-
Labels map[string]string `json:"labels"`
182-
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
183-
EnableForceDetach bool `json:"enableForceDetach"`
184-
DisableAuditLog bool `json:"disableAuditLog"`
185-
Debug bool `json:"debug"`
186-
Version *versionutils.Version `json:"version"`
187-
HTTPRequestTimeout string `json:"httpRequestTimeout"`
188-
NodeSelector map[string]string `json:"nodeSelector"`
189-
Tolerations []map[string]string `json:"tolerations"`
190-
ServiceAccountName string `json:"serviceAccountName"`
191-
ImagePullPolicy string `json:"imagePullPolicy"`
172+
DaemonsetName string `json:"daemonsetName"`
173+
TridentImage string `json:"tridentImage"`
174+
ImageRegistry string `json:"imageRegistry"`
175+
KubeletDir string `json:"kubeletDir"`
176+
LogFormat string `json:"logFormat"`
177+
LogLevel string `json:"logLevel"`
178+
LogWorkflows string `json:"logWorkflows"`
179+
LogLayers string `json:"logLayers"`
180+
ProbePort string `json:"probePort"`
181+
ImagePullSecrets []string `json:"imagePullSecrets"`
182+
Labels map[string]string `json:"labels"`
183+
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
184+
EnableForceDetach bool `json:"enableForceDetach"`
185+
DisableAuditLog bool `json:"disableAuditLog"`
186+
Debug bool `json:"debug"`
187+
Version *versionutils.Version `json:"version"`
188+
HTTPRequestTimeout string `json:"httpRequestTimeout"`
189+
NodeSelector map[string]string `json:"nodeSelector"`
190+
Tolerations []map[string]string `json:"tolerations"`
191+
TridentResourceSpec netappv1.ResourceSpec `json:"tridentResourceSpec"`
192+
RegistrarResourceSpec netappv1.ResourceSpec `json:"registrarSpec"`
193+
ServiceAccountName string `json:"serviceAccountName"`
194+
ImagePullPolicy string `json:"imagePullPolicy"`
192195
}
193196

194197
type TridentVersionPodYAMLArguments struct {

‎cli/k8s_client/yaml_factory.go‎

Lines changed: 65 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@ import (
99

1010
commonconfig "github.com/netapp/trident/config"
1111
. "github.com/netapp/trident/logging"
12+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
1213
"github.com/netapp/trident/utils"
14+
v1 "k8s.io/api/core/v1"
15+
"k8s.io/apimachinery/pkg/api/resource"
16+
k8syaml "sigs.k8s.io/yaml"
1317
)
1418

1519
const (
@@ -726,6 +730,25 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
726730
}
727731
}
728732

733+
defaultRegistrarResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
734+
Limits: v1.ResourceList{
735+
v1.ResourceMemory: resource.MustParse("100Mi"),
736+
},
737+
Requests: v1.ResourceList{
738+
v1.ResourceCPU: resource.MustParse("10m"),
739+
v1.ResourceMemory: resource.MustParse("40Mi"),
740+
},
741+
}}
742+
defaultTridentResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
743+
Limits: v1.ResourceList{
744+
v1.ResourceMemory: resource.MustParse("400Mi"),
745+
},
746+
Requests: v1.ResourceList{
747+
v1.ResourceCPU: resource.MustParse("10m"),
748+
v1.ResourceMemory: resource.MustParse("20Mi"),
749+
},
750+
}}
751+
729752
kubeletDir := strings.TrimRight(args.KubeletDir, "/")
730753
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
731754
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
@@ -745,6 +768,10 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
745768
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
746769
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
747770
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
771+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim(
772+
"DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResources))
773+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim(
774+
"DsTrident", args.TridentResourceSpec, defaultTridentResources))
748775
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
749776
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
750777
// Log before secrets are inserted into YAML.
@@ -799,6 +826,13 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
799826
}
800827
}
801828

829+
defaultRegistrarResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
830+
Requests: v1.ResourceList{},
831+
}}
832+
defaultTridentResources := netappv1.ResourceSpec{Resources: v1.ResourceRequirements{
833+
Requests: v1.ResourceList{},
834+
}}
835+
802836
kubeletDir := strings.TrimRight(args.KubeletDir, "/")
803837
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{TRIDENT_IMAGE}", args.TridentImage)
804838
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{DAEMONSET_NAME}", args.DaemonsetName)
@@ -819,6 +853,8 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
819853
daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
820854
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
821855
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
856+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "REGISTRAR_RESOURCE_CLAIMS", constructResourceClaim("DsRegistrar", args.RegistrarResourceSpec, defaultRegistrarResources))
857+
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "TRIDENT_RESOURCE_CLAIMS", constructResourceClaim("DsTrident", args.TridentResourceSpec, defaultTridentResources))
822858
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
823859
daemonSetYAML = replaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
824860

@@ -907,6 +943,7 @@ spec:
907943
failureThreshold: 5
908944
initialDelaySeconds: 10
909945
periodSeconds: 10
946+
{TRIDENT_RESOURCE_CLAIMS}
910947
env:
911948
- name: KUBE_NODE_NAME
912949
valueFrom:
@@ -948,6 +985,7 @@ spec:
948985
- "--v={SIDECAR_LOG_LEVEL}"
949986
- "--csi-address=$(ADDRESS)"
950987
- "--kubelet-registration-path=$(REGISTRATION_PATH)"
988+
{REGISTRAR_RESOURCE_CLAIMS}
951989
env:
952990
- name: ADDRESS
953991
value: /plugin/csi.sock
@@ -1137,12 +1175,7 @@ spec:
11371175
mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1
11381176
- name: csi-proxy-smb-pipe-v1beta1
11391177
mountPath: \\.\pipe\csi-proxy-smb-v1beta1
1140-
resources:
1141-
limits:
1142-
memory: 400Mi
1143-
requests:
1144-
cpu: 10m
1145-
memory: 20Mi
1178+
{TRIDENT_RESOURCE_CLAIMS}
11461179
- name: node-driver-registrar
11471180
image: {CSI_SIDECAR_REGISTRY}/csi-node-driver-registrar:v2.8.0
11481181
imagePullPolicy: {IMAGE_PULL_POLICY}
@@ -1174,12 +1207,7 @@ spec:
11741207
mountPath: C:\csi
11751208
- name: registration-dir
11761209
mountPath: C:\registration
1177-
resources:
1178-
limits:
1179-
memory: 200Mi
1180-
requests:
1181-
cpu: 10m
1182-
memory: 20Mi
1210+
{REGISTRAR_RESOURCE_CLAIMS}
11831211
- name: liveness-probe
11841212
volumeMounts:
11851213
- mountPath: C:\csi
@@ -1193,12 +1221,6 @@ spec:
11931221
env:
11941222
- name: CSI_ENDPOINT
11951223
value: unix:///csi/csi.sock
1196-
resources:
1197-
limits:
1198-
memory: 100Mi
1199-
requests:
1200-
cpu: 10m
1201-
memory: 40Mi
12021224
affinity:
12031225
nodeAffinity:
12041226
requiredDuringSchedulingIgnoredDuringExecution:
@@ -2509,6 +2531,31 @@ func constructNodeSelector(nodeLabels map[string]string) string {
25092531
return nodeSelector
25102532
}
25112533

2534+
func constructResourceClaim(
2535+
containerName string, resourceClaim netappv1.ResourceSpec, defaultResourceClaim netappv1.ResourceSpec,
2536+
) string {
2537+
userData, err := k8syaml.Marshal(resourceClaim)
2538+
if err != nil {
2539+
Log().WithField("container", containerName).Errorf("Cannot marshal resource claim: %s", err)
2540+
return ""
2541+
}
2542+
2543+
if err := k8syaml.Unmarshal(userData, &defaultResourceClaim); err != nil {
2544+
Log().WithField("container", containerName).Errorf(
2545+
"Cannot unmarshal provided resource claim into default resource claim: %s", err,
2546+
)
2547+
return ""
2548+
}
2549+
mergedData, err := k8syaml.Marshal(defaultResourceClaim)
2550+
2551+
if err != nil {
2552+
Log().WithField("container", containerName).Errorf("Cannot marshal resource claim: %s", err)
2553+
return ""
2554+
}
2555+
2556+
return string(mergedData)
2557+
}
2558+
25122559
func constructTolerations(tolerations []map[string]string) string {
25132560
var tolerationsString string
25142561

‎cli/k8s_client/yaml_factory_test.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ import (
1515
pspv1beta1 "k8s.io/api/policy/v1beta1"
1616
csiv1 "k8s.io/api/storage/v1"
1717
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
18+
"k8s.io/apimachinery/pkg/api/resource"
1819
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1920

2021
"github.com/netapp/trident/config"
22+
netappv1 "github.com/netapp/trident/operator/controllers/orchestrator/apis/netapp/v1"
2123
versionutils "github.com/netapp/trident/utils/version"
2224
)
2325

@@ -652,6 +654,43 @@ func TestGetCSIDaemonSetYAMLLinux_NodeSelectors(t *testing.T) {
652654
fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData))
653655
}
654656

657+
func TestGetCSIDaemonSetYAMLLinux_ResourceClaims(t *testing.T) {
658+
daemonsetArgs := &DaemonsetYAMLArguments{
659+
RegistrarResourceSpec: netappv1.ResourceSpec{},
660+
TridentResourceSpec: netappv1.ResourceSpec{
661+
Resources: v1.ResourceRequirements{
662+
Limits: v1.ResourceList{
663+
v1.ResourceMemory: resource.MustParse("125Mi"),
664+
},
665+
Requests: v1.ResourceList{
666+
v1.ResourceCPU: resource.MustParse("50m"),
667+
v1.ResourceMemory: resource.MustParse("50Mi"),
668+
},
669+
},
670+
},
671+
}
672+
expectedRegistrarResourceClaimString := "resources: {}\n"
673+
expectedTridentResourceClaimString := `
674+
resources:
675+
limits:
676+
memory: 125Mi
677+
requests:
678+
cpu: 50m
679+
memory: 50Mi
680+
`
681+
682+
yamlData := GetCSIDaemonSetYAMLLinux(daemonsetArgs)
683+
_, err := yaml.YAMLToJSON([]byte(yamlData))
684+
if err != nil {
685+
t.Fatalf("expected valid YAML, got %s", yamlData)
686+
}
687+
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
688+
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))
689+
690+
assert.Contains(t, yamlData, expectedTridentResourceClaimString,
691+
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
692+
}
693+
655694
func TestGetCSIDaemonSetYAMLLinux_Tolerations(t *testing.T) {
656695
daemonsetArgs := &DaemonsetYAMLArguments{
657696
Tolerations: []map[string]string{
@@ -800,6 +839,56 @@ func TestGetCSIDaemonSetYAMLWindowsImagePullPolicy(t *testing.T) {
800839
}
801840
}
802841

842+
func TestGetCSIDaemonSetYAMWindows_ResourceClaims(t *testing.T) {
843+
versions := []string{"1.26.0"}
844+
registrarResourceSpec := netappv1.ResourceSpec{}
845+
tridentResourceSpec := netappv1.ResourceSpec{
846+
Resources: v1.ResourceRequirements{
847+
Limits: v1.ResourceList{
848+
v1.ResourceMemory: resource.MustParse("125Mi"),
849+
},
850+
Requests: v1.ResourceList{
851+
v1.ResourceCPU: resource.MustParse("50m"),
852+
},
853+
},
854+
}
855+
856+
expectedRegistrarResourceClaimString := `
857+
resources:
858+
limits:
859+
memory: 100Mi
860+
requests:
861+
cpu: 10m
862+
memory: 40Mi
863+
`
864+
expectedTridentResourceClaimString := `
865+
resources:
866+
limits:
867+
memory: 125Mi
868+
requests:
869+
cpu: 50m
870+
memory: 20Mi
871+
`
872+
for _, versionString := range versions {
873+
version := versionutils.MustParseSemantic(versionString)
874+
daemonsetArgs := &DaemonsetYAMLArguments{
875+
Version: version,
876+
RegistrarResourceSpec: registrarResourceSpec,
877+
TridentResourceSpec: tridentResourceSpec,
878+
}
879+
yamlData := GetCSIDaemonSetYAMLWindows(daemonsetArgs)
880+
_, err := yaml.YAMLToJSON([]byte(yamlData))
881+
if err != nil {
882+
t.Fatalf("expected valid YAML, got %s", yamlData)
883+
}
884+
assert.Contains(t, yamlData, expectedRegistrarResourceClaimString,
885+
fmt.Sprintf("expected empty registrar resource claims in final YAML: %s", yamlData))
886+
887+
assert.Contains(t, yamlData, expectedTridentResourceClaimString,
888+
fmt.Sprintf("expected trident resource claims in final YAML: %s", yamlData))
889+
}
890+
}
891+
803892
func TestConstructNodeSelector(t *testing.T) {
804893
nodeSelMap := map[string]string{"node-label-name": "master"}
805894

@@ -809,6 +898,54 @@ func TestConstructNodeSelector(t *testing.T) {
809898
assert.Equal(t, expectedNodeSelString, result)
810899
}
811900

901+
func TestConstructResourceClaim(t *testing.T) {
902+
defaultResourceClaim := netappv1.ResourceSpec{
903+
Resources: v1.ResourceRequirements{},
904+
}
905+
resourceClaim := netappv1.ResourceSpec{
906+
Resources: v1.ResourceRequirements{
907+
Limits: v1.ResourceList{
908+
v1.ResourceCPU: resource.MustParse("100m"),
909+
v1.ResourceMemory: resource.MustParse("50Mi"),
910+
},
911+
Requests: v1.ResourceList{
912+
v1.ResourceCPU: resource.MustParse("20m"),
913+
v1.ResourceMemory: resource.MustParse("20Mi"),
914+
},
915+
},
916+
}
917+
918+
expectedResourceClaimString := `resources:
919+
limits:
920+
cpu: 100m
921+
memory: 50Mi
922+
requests:
923+
cpu: 20m
924+
memory: 20Mi
925+
`
926+
927+
result := constructResourceClaim("test", resourceClaim, defaultResourceClaim)
928+
assert.Equal(t, expectedResourceClaimString, result)
929+
930+
resourceClaim = netappv1.ResourceSpec{
931+
Resources: v1.ResourceRequirements{
932+
Limits: v1.ResourceList{
933+
v1.ResourceCPU: resource.MustParse("100m"),
934+
},
935+
},
936+
}
937+
938+
expectedResourceClaimString = "resources:\n limits:\n cpu: 100m\n"
939+
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
940+
assert.Equal(t, expectedResourceClaimString, result)
941+
942+
resourceClaim = netappv1.ResourceSpec{}
943+
944+
expectedResourceClaimString = "resources: {}\n"
945+
result = constructResourceClaim("test", resourceClaim, defaultResourceClaim)
946+
assert.Equal(t, expectedResourceClaimString, result)
947+
}
948+
812949
func TestGetNamespaceYAML(t *testing.T) {
813950
expected := v1.Namespace{
814951
TypeMeta: metav1.TypeMeta{

0 commit comments

Comments
 (0)