CNF-24269 Add Core control-plane PerformanceProfile

Add PerformanceProfile and Tuned configuration for Core control-plane MCP. Number of
reserved/isolated CPUs, hugepages, and systemReserved memory are all variable content.

The kube-compare reference uses the same PerformanceProfile template for both control plane
and worker profiles with differences keyed off the machineConfigPoolSelector.

Signed-off-by: Ian Miller <imiller@redhat.com>

Author: imiller0

telco-core/configuration/core-overlay.yaml

@@ -132,13 +132,25 @@ policies:
             nodeSelector:
               node-role.kubernetes.io/worker-3: ""
 
+      # Control plane PerformanceProfile
+      - path: reference-crs/required/performance/PerformanceProfile-control-plane.yaml
+        patches:
+        - spec:
+            cpu:
+              reserved: '{{hub fromConfigMap "" "hw-types" "control-plane-reserved" | toLiteral hub}}'
+              isolated: '{{hub fromConfigMap "" "hw-types" "control-plane-isolated" | toLiteral hub}}'
+
       # TuneD performance patches for core
       - path: reference-crs/required/performance/TunedPerformancePatch.yaml
         patches:
         - metadata:
             name: telco-core-performance-patch
           spec:
             recommend:
+              - machineConfigLabels:
+                  machineconfiguration.openshift.io/role: master
+                priority: 18
+                profile: control-plane-profile
               - machineConfigLabels:
                   machineconfiguration.openshift.io/role: worker-1
                 priority: 18

telco-core/configuration/reference-crs-kube-compare/compare_ignore

@@ -38,3 +38,6 @@ custom-manifests/subscription-validator.yaml
 
 # This is an object-template-raw used only to validate pre-caching state
 custom-manifests/precache-validator.yaml
+
+# Control plane PerformanceProfile is merged into the main PerformanceProfile.yaml template
+required/performance/PerformanceProfile-control-plane.yaml

telco-core/configuration/reference-crs-kube-compare/required/performance/PerformanceProfile.yaml

@@ -3,10 +3,28 @@
 apiVersion: performance.openshift.io/v2
 kind: PerformanceProfile
 metadata:
-  name: {{ .metadata.name }}
+  {{- $isControlPlane := false }}
+  {{- if hasKey .spec "machineConfigPoolSelector" }}
+    {{- if hasKey .spec.machineConfigPoolSelector "pools.operator.machineconfiguration.openshift.io/master" }}
+      {{- $isControlPlane = true }}
+    {{- end }}
+  {{- end }}
+  name: {{ if $isControlPlane }}control-plane-profile{{ else }}{{ .metadata.name }}{{ end }}
   annotations:
+    {{- if $isControlPlane }}
+    # systemReserved: when used, it should be tailored for each environment.
+    {{- $config := (index (.metadata.annotations | default dict) "kubeletconfig.experimental") | fromJson }}
+    {{- if hasKey $config "allowedUnsafeSysctls" }}
+      {{- if ne (len $config.allowedUnsafeSysctls) 0 }}
+        {{- fail "Control plane PerformanceProfile must not have any allowedUnsafeSysctls" }}
+      {{- end }}
+    {{- end }}
+    kubeletconfig.experimental: |
+      {
+       "systemReserved":{{ $config.systemReserved | toJson }}
+      }
+    {{- else }}
     # Some pods want the kernel stack to ignore IPv6 router Advertisement.
-    {{- if .metadata }}
       {{ $allowedSysctls := (list
           "net.ipv6.conf.default.accept_ra"
           "net.ipv6.conf.all.disable_ipv6"
@@ -30,17 +48,10 @@ spec:
       "intel_idle.max_cstate=0"
       ) ) }}
   cpu:
-    # node0 CPUs: 0-17,36-53
-    # node1 CPUs: 18-34,54-71
-    # siblings: (0,36), (1,37)...
-    # we want to reserve the first Core of each NUMA socket
-    #
-    # no CPU left behind! all-cpus == isolated + reserved
-    isolated: {{ .spec.cpu.isolated }} # eg 1-17,19-35,37-53,55-71
-    reserved: {{ .spec.cpu.reserved }} # eg 0,18,36,54
-  # Guaranteed QoS pods will disable IRQ balancing for cores allocated to the pod.
-  # default value of globallyDisableIrqLoadBalancing is false
+    isolated: {{ .spec.cpu.isolated }}
+    reserved: {{ .spec.cpu.reserved }}
   globallyDisableIrqLoadBalancing: false
+  {{- if hasKey .spec "hugepages" }}
   hugepages:
     defaultHugepagesSize: 1G
     pages:
@@ -51,20 +62,17 @@ spec:
           node: {{ .node }}
           {{- end }}
         {{- end }}
+  {{- end }}
   {{- if hasKey .spec "kernelPageSize" }}
   kernelPageSize: {{ .spec.kernelPageSize }}
   {{- end }}
   {{- if hasKey .spec "machineConfigPoolSelector" }}
   machineConfigPoolSelector:
     {{- .spec.machineConfigPoolSelector | toYaml | nindent 4 }}
-#    # For SNO: machineconfiguration.openshift.io/role: 'master'
-#    pools.operator.machineconfiguration.openshift.io/worker: ''
   {{- end }}
   {{- if hasKey .spec "nodeSelector" }}
   nodeSelector:
     {{- .spec.nodeSelector | toYaml | nindent 4 }}
-#    # For SNO: node-role.kubernetes.io/master: ""
-#    node-role.kubernetes.io/worker: ""
   {{- end }}
   workloadHints:
     realTime: false

telco-core/configuration/reference-crs/required/performance/PerformanceProfile-control-plane.yaml

@@ -0,0 +1,49 @@
+# required
+# count: 1
+---
+apiVersion: performance.openshift.io/v2
+kind: PerformanceProfile
+metadata:
+  name: control-plane-profile
+  annotations:
+    # systemReserved: when used, it should be tailored for each environment.
+    kubeletconfig.experimental: |
+      {
+       "systemReserved":{"memory":"11Gi"}
+      }
+spec:
+  # Optional kernel arguments:
+  #   To enable acpi_idle CPUIdle driver
+  #     - intel_idle.max_cstate=0
+  additionalKernelArgs:
+  - module_blacklist=irdma
+  cpu:
+    # siblings: (0,36), (1,37)...
+    # we want to reserve 8 cores (16 CPUs with HT)
+    #
+    # no CPU left behind! all-cpus == isolated + reserved
+    isolated: 8-35,44-71
+    reserved: 0-7,36-43
+  # default value of globallyDisableIrqLoadBalancing is false
+  globallyDisableIrqLoadBalancing: false
+  # Hugepages may be configured as needed when using schedulable control planes
+  #hugepages:
+  #  defaultHugepagesSize: 1G
+  #  pages:
+  #  - count: 64
+  #    size: 1G
+  machineConfigPoolSelector:
+    pools.operator.machineconfiguration.openshift.io/master: ''
+  nodeSelector:
+    node-role.kubernetes.io/control-plane: ""
+  workloadHints:
+    realTime: false
+    highPowerConsumption: false
+    perPodPowerManagement: true
+  realTimeKernel:
+    enabled: false
+  numa:
+    # All guaranteed QoS containers get resources from a single NUMA node
+    topologyPolicy: "single-numa-node"
+  net:
+    userLevelNetworking: false

telco-core/configuration/reference-crs/required/performance/TunedPerformancePatch.yaml

@@ -13,6 +13,13 @@ spec:
       data: |
         [main]
         summary=Top-level core performance tuning overrides
+    - name: control-plane-profile
+      data: |
+        [main]
+        summary=Control plane core performance tuning overrides
+        include=openshift-node-performance-control-plane-profile
+        [sysctl]
+        kernel.panic_on_unrecovered_nmi=1
     - name: worker-profile-1
       data: |
         [main]

telco-core/configuration/template-values/hw-types.yaml

@@ -5,6 +5,9 @@ metadata:
   name: hw-types
   namespace: ztp-core-policies
 data:
+  control-plane-reserved: "0-7,36-43"
+  control-plane-isolated: "8-35,44-71"
+
   role-worker-1-reserved: "0-1,52-53"
   role-worker-1-isolated: "2-51,54-103"
   role-worker-1-hugepg-cnt: "4"