From 7c2e3d12cf9da0b69b09b9b8d0240393ed1ce769 Mon Sep 17 00:00:00 2001 From: Keerthana Chennagoni Date: Mon, 22 Jun 2026 14:21:43 -0400 Subject: [PATCH 1/2] CNF-22194 Core: ensure one copy of MachineConfigs in Core RDS --- telco-core/configuration/Makefile | 5 +- telco-core/configuration/compare.sh | 102 +++++++++++++++++- telco-core/configuration/core-overlay.yaml | 9 +- .../reference-crs-kube-compare/compare_ignore | 20 +++- .../reference-crs-kube-compare/metadata.yaml | 15 +-- .../tap_cni/mc_rootless_pods_selinux.yaml | 5 +- .../control-plane-load-kernel-modules.yaml | 1 + .../other/worker-load-kernel-modules.yaml | 1 + .../reference-crs/custom-manifests/README.md | 23 ++-- .../custom-manifests/mcp-worker-1.yaml | 4 +- .../custom-manifests/mcp-worker-2.yaml | 4 +- .../custom-manifests/mcp-worker-3.yaml | 4 +- .../tap_cni/mc_rootless_pods_selinux.yaml | 27 ----- .../control-plane-load-kernel-modules.yaml | 38 ------- .../optional/other/kdump-master.yaml | 19 ---- .../optional/other/kdump-worker.yaml | 19 ---- .../other/mount_namespace_config_master.yaml | 15 --- .../other/mount_namespace_config_worker.yaml | 15 --- .../optional/other/sctp_module_mc.yaml | 25 ----- .../other/worker-load-kernel-modules.yaml | 38 ------- telco-core/install/README.md | 65 ++++++++--- telco-core/install/custom-manifests/README.md | 9 ++ .../example-standard-clusterinstance.yaml | 1 - .../install/extra-manifests/mcp-worker-1.yaml | 18 ++++ .../install/extra-manifests/mcp-worker-2.yaml | 18 ++++ .../install/extra-manifests/mcp-worker-3.yaml | 18 ++++ .../extra-manifests/sctp_module_mc.yaml | 21 ++-- telco-core/install/kustomization.yaml | 29 +++++ 28 files changed, 299 insertions(+), 269 deletions(-) delete mode 100644 telco-core/configuration/reference-crs/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/control-plane-load-kernel-modules.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/kdump-master.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/kdump-worker.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/mount_namespace_config_master.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/mount_namespace_config_worker.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/sctp_module_mc.yaml delete mode 100644 telco-core/configuration/reference-crs/optional/other/worker-load-kernel-modules.yaml create mode 100644 telco-core/install/custom-manifests/README.md create mode 100644 telco-core/install/extra-manifests/mcp-worker-1.yaml create mode 100644 telco-core/install/extra-manifests/mcp-worker-2.yaml create mode 100644 telco-core/install/extra-manifests/mcp-worker-3.yaml create mode 100644 telco-core/install/kustomization.yaml diff --git a/telco-core/configuration/Makefile b/telco-core/configuration/Makefile index c64604250..d6105b3d9 100644 --- a/telco-core/configuration/Makefile +++ b/telco-core/configuration/Makefile @@ -1,6 +1,9 @@ .PHONY: check -check: compare_crs missing_crs +check: compare_crs missing_crs compare_extra_manifests +.PHONY: compare_extra_manifests +compare_extra_manifests: + ./compare.sh --check-extra-manifests .PHONY: compare_crs compare_crs: diff --git a/telco-core/configuration/compare.sh b/telco-core/configuration/compare.sh index a34cf2741..e2f727f48 100755 --- a/telco-core/configuration/compare.sh +++ b/telco-core/configuration/compare.sh @@ -112,13 +112,101 @@ sync_cr() { git diff --cached --stat --exit-code } +# Install-time MachineConfig CRs live only under telco-core/install/extra-manifests. +# kube-compare-reference holds validation templates; non-templated files must match install. +compare_install_extra_manifests() { + local root fail=0 + root="$(git rev-parse --show-toplevel)" + local install="${root}/telco-core/install/extra-manifests" + local kubecmp="${root}/telco-core/configuration/reference-crs-kube-compare" + local -a pairs=( + "control-plane-load-kernel-modules.yaml:optional/other/control-plane-load-kernel-modules.yaml" + "worker-load-kernel-modules.yaml:optional/other/worker-load-kernel-modules.yaml" + "mount_namespace_config_master.yaml:optional/other/mount_namespace_config_master.yaml" + "mount_namespace_config_worker.yaml:optional/other/mount_namespace_config_worker.yaml" + "kdump-master.yaml:optional/other/kdump-master.yaml" + "kdump-worker.yaml:optional/other/kdump-worker.yaml" + "mc_rootless_pods_selinux.yaml:optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml" + ) + local pair inst ref + for pair in "${pairs[@]}"; do + inst="${pair%%:*}" + ref="${pair##*:}" + if ! diff -u "${install}/${inst}" "${kubecmp}/${ref}"; then + echo "ERROR: install/extra-manifests/${inst} differs from kube-compare ${ref}" >&2 + fail=1 + fi + done + local sctp="${install}/sctp_module_mc.yaml" + if ! grep -q '{{' "${kubecmp}/optional/other/sctp_module_mc.yaml"; then + echo "ERROR: expected templating in optional/other/sctp_module_mc.yaml" >&2 + fail=1 + fi + if ! grep -q 'version: 3.2.0' "${sctp}"; then + echo "ERROR: ${sctp} must use ignition version 3.2.0" >&2 + fail=1 + fi + if ! grep -q 'source: data:,sctp' "${sctp}"; then + echo "ERROR: ${sctp} must load sctp via data:,sctp" >&2 + fail=1 + fi + if grep -q 'version: 2.2.0' "${sctp}" || grep -q 'filesystem: root' "${sctp}"; then + echo "ERROR: ${sctp} must not use legacy ignition 2.2.0 / filesystem fields" >&2 + fail=1 + fi + return $fail +} + +# PolicyGenerator manifest paths must stay under telco-core/configuration/ (no ../). +# MCP CRs are duplicated here and must match install/extra-manifests. +compare_install_custom_manifests_mcp() { + local root fail=0 f + root="$(git rev-parse --show-toplevel)" + local install="${root}/telco-core/install/extra-manifests" + local custom="${root}/telco-core/configuration/reference-crs/custom-manifests" + for f in mcp-worker-1.yaml mcp-worker-2.yaml mcp-worker-3.yaml; do + if ! diff -u "${install}/${f}" "${custom}/${f}"; then + echo "ERROR: install/extra-manifests/${f} differs from reference-crs/custom-manifests/${f}" >&2 + fail=1 + fi + done + return $fail +} + +check_no_machineconfig_in_reference_crs() { + local root fail=0 f + root="$(git rev-parse --show-toplevel)" + while IFS= read -r f; do + echo "ERROR: MachineConfig must use install/extra-manifests, not reference-crs: ${f}" >&2 + fail=1 + done < <(grep -rl '^kind: MachineConfig$' "${root}/telco-core/configuration/reference-crs" 2>/dev/null || true) + return $fail +} + +run_extra_manifest_checks() { + local status=0 + echo "Checking install/extra-manifests alignment with kube-compare-reference..." + compare_install_extra_manifests || status=1 + echo "Checking install/extra-manifests MCPs match reference-crs/custom-manifests..." + compare_install_custom_manifests_mcp || status=1 + echo "Checking reference-crs does not contain MachineConfig CRs..." + check_no_machineconfig_in_reference_crs || status=1 + if [[ $status -eq 0 ]]; then + echo "extra-manifest checks: OK" + fi + return $status +} + usage() { - echo "$(basename "$0") [--sync] sourceDir renderDir" + echo "$(basename "$0") [--sync] sourceDir renderDir ignoreFile" + echo "$(basename "$0") --check-extra-manifests" echo - echo "Compares the rendered reference-based CRs to the CRs in the compare directory" + echo "Compares the rendered reference-based CRs to the CRs in the compare directory," + echo "or validates install/extra-manifests vs kube-compare-reference." } DOSYNC=0 +CHECK_EXTRA_MANIFESTS=0 for arg in "$@"; do case "$arg" in -h | --help) @@ -129,8 +217,18 @@ for arg in "$@"; do DOSYNC=1 shift ;; + --check-extra-manifests) + CHECK_EXTRA_MANIFESTS=1 + shift + ;; esac done + +if [[ $CHECK_EXTRA_MANIFESTS == 1 ]]; then + run_extra_manifest_checks + exit $? +fi + SOURCEDIR=$1 if [[ ! -d $SOURCEDIR ]]; then echo "No such source directory $SOURCEDIR" diff --git a/telco-core/configuration/core-overlay.yaml b/telco-core/configuration/core-overlay.yaml index ec625c5c6..a8f784283 100644 --- a/telco-core/configuration/core-overlay.yaml +++ b/telco-core/configuration/core-overlay.yaml @@ -26,13 +26,8 @@ policies: - mirrors: - example.com/sample-path source: nomatch.io/sample-path - - path: reference-crs/optional/other/sctp_module_mc.yaml - complianceType: mustonlyhave - - path: reference-crs/optional/other/worker-load-kernel-modules.yaml - complianceType: mustonlyhave - - path: reference-crs/optional/other/control-plane-load-kernel-modules.yaml - complianceType: mustonlyhave - - path: reference-crs/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml + # MachineConfig CRs are applied at install time from telco-core/install/extra-manifests + # and kept in sync at day-N via the Hub extra-manifests policy (see telco-hub ztp-policies). # Cert-manager configuration (optional component) # - path: reference-crs/optional/cert-manager/certManagerClusterIssuer.yaml diff --git a/telco-core/configuration/reference-crs-kube-compare/compare_ignore b/telco-core/configuration/reference-crs-kube-compare/compare_ignore index a78b85a59..dbb45530e 100644 --- a/telco-core/configuration/reference-crs-kube-compare/compare_ignore +++ b/telco-core/configuration/reference-crs-kube-compare/compare_ignore @@ -24,15 +24,25 @@ infrastructure.yaml optional/networking/sriov/SriovNetworkPoolConfig.yaml required/networking/metallb/service.yaml -# These are examples only. Customer will provide their own -# based on cluster topology and desired MCP partitioning. -custom-manifests/mcp-worker-1.yaml -custom-manifests/mcp-worker-2.yaml -custom-manifests/mcp-worker-3.yaml +# MachineConfig CRs live under telco-core/install/extra-manifests (install + Hub policy). +# kube-compare keeps optional paths for cluster-compare; they are not duplicated in reference-crs. +optional/other/control-plane-load-kernel-modules.yaml +optional/other/worker-load-kernel-modules.yaml +optional/other/sctp_module_mc.yaml +optional/other/mount_namespace_config_master.yaml +optional/other/mount_namespace_config_worker.yaml +optional/other/kdump-master.yaml +optional/other/kdump-worker.yaml +optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml # Certificate Copying CRs optional/other/server-cert-openshift-config-copy.yaml +# MachineConfigPool CRs for PolicyGenerator (canonical copies under install/extra-manifests). +custom-manifests/mcp-worker-1.yaml +custom-manifests/mcp-worker-2.yaml +custom-manifests/mcp-worker-3.yaml + # This is an object-template-raw used only to validate subscription state custom-manifests/subscription-validator.yaml diff --git a/telco-core/configuration/reference-crs-kube-compare/metadata.yaml b/telco-core/configuration/reference-crs-kube-compare/metadata.yaml index a0c9d508d..198052b55 100644 --- a/telco-core/configuration/reference-crs-kube-compare/metadata.yaml +++ b/telco-core/configuration/reference-crs-kube-compare/metadata.yaml @@ -91,7 +91,6 @@ parts: - name: optional-networking anyOf: - path: optional/networking/networkAttachmentDefinition.yaml - - path: optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml - path: required/networking/metallb/community.yaml - path: optional/networking/nodeNetworkConfigurationPolicy.yaml - name: networking-firewall @@ -159,17 +158,17 @@ parts: - name: other description: |- https://docs.redhat.com/en/documentation/openshift_container_platform/4.22/html/scalability_and_performance/telco-core-ref-design-specs#node-configuration-crs + + Reference MachineConfig CRs are validated from telco-core/install/extra-manifests + via compare.sh --check-extra-manifests, not from reference-crs. components: - name: other anyOf: - - path: optional/other/control-plane-load-kernel-modules.yaml - path: optional/other/monitoring-config-cm.yaml config: perField: - pathToKey: data."config.yaml" inlineDiffFunc: capturegroups - - path: optional/other/worker-load-kernel-modules.yaml - - path: optional/other/sctp_module_mc.yaml - name: precache anyOf: - path: optional/other/precache/controlplane-olm-pinned-images.yaml @@ -198,14 +197,6 @@ parts: - name: other anyOf: - path: optional/tuning/control-plane-system-reserved.yaml - - name: kdump-configuration - allOf: - - path: optional/other/kdump-master.yaml - - path: optional/other/kdump-worker.yaml - - name: mount-namespace-configuration - allOf: - - path: optional/other/mount_namespace_config_master.yaml - - path: optional/other/mount_namespace_config_worker.yaml - name: optional-cert-manager description: |- Cert-manager operator for automated certificate management diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml index b08bb52b9..42d3af1cc 100644 --- a/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml +++ b/telco-core/configuration/reference-crs-kube-compare/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml @@ -1,3 +1,4 @@ +--- apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: @@ -14,12 +15,12 @@ spec: [Unit] Description=Set SELinux boolean for tap cni plugin Before=kubelet.service - + [Service] Type=oneshot ExecStart=/sbin/setsebool container_use_devices=on RemainAfterExit=true - + [Install] WantedBy=multi-user.target graphical.target enabled: true diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml index 33a800559..177565995 100644 --- a/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml +++ b/telco-core/configuration/reference-crs-kube-compare/optional/other/control-plane-load-kernel-modules.yaml @@ -1,5 +1,6 @@ # optional # count: 1 +--- apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: diff --git a/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml b/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml index 2576ccaa8..42a21308c 100644 --- a/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml +++ b/telco-core/configuration/reference-crs-kube-compare/optional/other/worker-load-kernel-modules.yaml @@ -1,5 +1,6 @@ # optional # count: 1 +--- apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: diff --git a/telco-core/configuration/reference-crs/custom-manifests/README.md b/telco-core/configuration/reference-crs/custom-manifests/README.md index 509c7cdfa..8fa52d664 100644 --- a/telco-core/configuration/reference-crs/custom-manifests/README.md +++ b/telco-core/configuration/reference-crs/custom-manifests/README.md @@ -1,10 +1,15 @@ # Custom CRs -This directory is a placeholder for additional custom CRs which are -outside the scope of the reference CRs. - -## Reference MCPs -The MCP CRs contained here are examples for a multi-mcp cluster and -are used to build policies which accelerate initial rollout of -configuration by increasing maxUnavailable and manipulating the MCP -pause. The MCPs should be created during initial cluster installation -in a paused state with maxUnavailable of 100%. + +This directory holds CRs referenced by PolicyGenerator manifests under +`telco-core/configuration/`. Paths must stay under that directory tree because +the PolicyGenerator kustomize plugin rejects manifest paths outside it (no `../`). + +## MachineConfigPool examples + +Reference `MachineConfigPool` CRs (`mcp-worker-1.yaml` through `mcp-worker-3.yaml`) +are duplicated here for day-N policies (`core-finish`, `core-upgrade`, +`core-upgrade-finish`). The canonical install-time copies live under +`telco-core/install/extra-manifests/` and must remain identical; `compare.sh +--check-extra-manifests` enforces that. + +Other custom content (for example `subscription-validator.yaml`) also belongs here. diff --git a/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-1.yaml b/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-1.yaml index 6a00d0e0b..f590e53fb 100644 --- a/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-1.yaml +++ b/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-1.yaml @@ -11,8 +11,8 @@ spec: - key: machineconfiguration.openshift.io/role operator: In values: [ worker, worker-1 ] - paused: false - maxUnavailable: 1 + paused: true + maxUnavailable: 100% nodeSelector: matchLabels: node-role.kubernetes.io/worker-1: "" diff --git a/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-2.yaml b/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-2.yaml index 5d24c9299..39aa39b4e 100644 --- a/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-2.yaml +++ b/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-2.yaml @@ -11,8 +11,8 @@ spec: - key: machineconfiguration.openshift.io/role operator: In values: [ worker, worker-2 ] - paused: false - maxUnavailable: 1 + paused: true + maxUnavailable: 100% nodeSelector: matchLabels: node-role.kubernetes.io/worker-2: "" diff --git a/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-3.yaml b/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-3.yaml index d40972697..758dc6fb3 100644 --- a/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-3.yaml +++ b/telco-core/configuration/reference-crs/custom-manifests/mcp-worker-3.yaml @@ -11,8 +11,8 @@ spec: - key: machineconfiguration.openshift.io/role operator: In values: [ worker, worker-3 ] - paused: false - maxUnavailable: 1 + paused: true + maxUnavailable: 100% nodeSelector: matchLabels: node-role.kubernetes.io/worker-3: "" diff --git a/telco-core/configuration/reference-crs/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml b/telco-core/configuration/reference-crs/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml deleted file mode 100644 index 42d3af1cc..000000000 --- a/telco-core/configuration/reference-crs/optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 99-worker-setsebool -spec: - config: - ignition: - version: 3.2.0 - systemd: - units: - - contents: | - [Unit] - Description=Set SELinux boolean for tap cni plugin - Before=kubelet.service - - [Service] - Type=oneshot - ExecStart=/sbin/setsebool container_use_devices=on - RemainAfterExit=true - - [Install] - WantedBy=multi-user.target graphical.target - enabled: true - name: setsebool.service diff --git a/telco-core/configuration/reference-crs/optional/other/control-plane-load-kernel-modules.yaml b/telco-core/configuration/reference-crs/optional/other/control-plane-load-kernel-modules.yaml deleted file mode 100644 index cb20cdcbd..000000000 --- a/telco-core/configuration/reference-crs/optional/other/control-plane-load-kernel-modules.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# optional -# count: 1 ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 40-load-kernel-modules-control-plane -spec: - config: - # Release info found in https://github.com/coreos/butane/releases - ignition: - version: 3.2.0 - storage: - files: - - contents: - source: data:, - mode: 420 - overwrite: true - path: /etc/modprobe.d/kernel-blacklist.conf - - contents: - # Contents - # ip_gre - # nf_tables - # nf_conntrack - # nft_ct - # nft_limit - # nft_log - # nft_nat - # nft_chain_nat - # nf_reject_ipv4 - # nf_reject_ipv6 - # nfnetlink_log - source: data:text/plain;charset=utf-8;base64,aXBfZ3JlCm5mX3RhYmxlcwpuZl9jb25udHJhY2sKbmZ0X2N0Cm5mdF9saW1pdApuZnRfbG9nCm5mdF9uYXQKbmZ0X2NoYWluX25hdApuZl9yZWplY3RfaXB2NApuZl9yZWplY3RfaXB2NgpuZm5ldGxpbmtfbG9nCg== - mode: 420 - overwrite: true - path: /etc/modules-load.d/kernel-load.conf diff --git a/telco-core/configuration/reference-crs/optional/other/kdump-master.yaml b/telco-core/configuration/reference-crs/optional/other/kdump-master.yaml deleted file mode 100644 index 89ffc8f07..000000000 --- a/telco-core/configuration/reference-crs/optional/other/kdump-master.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Automatically generated by extra-manifests-builder -# Do not make changes directly. ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 06-kdump-enable-master -spec: - config: - ignition: - version: 3.2.0 - systemd: - units: - - enabled: true - name: kdump.service - kernelArguments: - - crashkernel=512M diff --git a/telco-core/configuration/reference-crs/optional/other/kdump-worker.yaml b/telco-core/configuration/reference-crs/optional/other/kdump-worker.yaml deleted file mode 100644 index f55ebc1cf..000000000 --- a/telco-core/configuration/reference-crs/optional/other/kdump-worker.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Automatically generated by extra-manifests-builder -# Do not make changes directly. ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 06-kdump-enable-worker -spec: - config: - ignition: - version: 3.2.0 - systemd: - units: - - enabled: true - name: kdump.service - kernelArguments: - - crashkernel=512M diff --git a/telco-core/configuration/reference-crs/optional/other/mount_namespace_config_master.yaml b/telco-core/configuration/reference-crs/optional/other/mount_namespace_config_master.yaml deleted file mode 100644 index f20655476..000000000 --- a/telco-core/configuration/reference-crs/optional/other/mount_namespace_config_master.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: master - name: 99-kubens-master -spec: - config: - ignition: - version: 3.2.0 - systemd: - units: - - enabled: true - name: kubens.service diff --git a/telco-core/configuration/reference-crs/optional/other/mount_namespace_config_worker.yaml b/telco-core/configuration/reference-crs/optional/other/mount_namespace_config_worker.yaml deleted file mode 100644 index 65a496fd1..000000000 --- a/telco-core/configuration/reference-crs/optional/other/mount_namespace_config_worker.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 99-kubens-worker -spec: - config: - ignition: - version: 3.2.0 - systemd: - units: - - enabled: true - name: kubens.service diff --git a/telco-core/configuration/reference-crs/optional/other/sctp_module_mc.yaml b/telco-core/configuration/reference-crs/optional/other/sctp_module_mc.yaml deleted file mode 100644 index 71e26380b..000000000 --- a/telco-core/configuration/reference-crs/optional/other/sctp_module_mc.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# optional -# count: 1 ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - name: load-sctp-module - labels: - machineconfiguration.openshift.io/role: worker -spec: - config: - ignition: - version: 3.2.0 - storage: - files: - - path: /etc/modprobe.d/sctp-blacklist.conf - mode: 420 - overwrite: true - contents: - source: data:, - - path: /etc/modules-load.d/sctp-load.conf - mode: 420 - overwrite: true - contents: - source: data:,sctp diff --git a/telco-core/configuration/reference-crs/optional/other/worker-load-kernel-modules.yaml b/telco-core/configuration/reference-crs/optional/other/worker-load-kernel-modules.yaml deleted file mode 100644 index e2b8af9f7..000000000 --- a/telco-core/configuration/reference-crs/optional/other/worker-load-kernel-modules.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# optional -# count: 1 ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: worker - name: 40-load-kernel-modules-worker -spec: - config: - # Release info found in https://github.com/coreos/butane/releases - ignition: - version: 3.2.0 - storage: - files: - - contents: - source: data:, - mode: 420 - overwrite: true - path: /etc/modprobe.d/kernel-blacklist.conf - - contents: - # Contents - # ip_gre - # nf_tables - # nf_conntrack - # nft_ct - # nft_limit - # nft_log - # nft_nat - # nft_chain_nat - # nf_reject_ipv4 - # nf_reject_ipv6 - # nfnetlink_log - source: data:text/plain;charset=utf-8;base64,aXBfZ3JlCm5mX3RhYmxlcwpuZl9jb25udHJhY2sKbmZ0X2N0Cm5mdF9saW1pdApuZnRfbG9nCm5mdF9uYXQKbmZ0X2NoYWluX25hdApuZl9yZWplY3RfaXB2NApuZl9yZWplY3RfaXB2NgpuZm5ldGxpbmtfbG9nCg== - mode: 420 - overwrite: true - path: /etc/modules-load.d/kernel-load.conf diff --git a/telco-core/install/README.md b/telco-core/install/README.md index 2a4148d26..b8b98a845 100644 --- a/telco-core/install/README.md +++ b/telco-core/install/README.md @@ -11,10 +11,27 @@ general pattern shown here is recommended. The contents of this directory fall into 3 categories. Each of these are described in more detail in the following sections: -- example-standard.yaml -- A SiteConfig CR which defines the cluster +- example-standard-clusterinstance.yaml -- A ClusterInstance CR for installation +- kustomization.yaml -- Builds `extra-manifests-configmap` from `extra-manifests/` - extra-manifests -- Additional reference CRs to apply to the cluster during installation - custom-manifests -- Additional custom/user specific CRs to apply to the cluster during installation +## ClusterInstance and extra-manifests ConfigMap + +Reference `MachineConfig` and `MachineConfigPool` CRs are packaged for install using +`kustomization.yaml`. The `configMapGenerator` lists every file under +`extra-manifests/`; `example-standard-clusterinstance.yaml` references the result +via `spec.extraManifestsRefs` (`extra-manifests-configmap`). + +```bash +kubectl apply -k telco-core/install/ +``` + +At day-N, the Hub **extra-manifests** policy keeps the cluster aligned with the +ConfigMap content (see `telco-hub/.../ztp-policies/extra-manifests-policy.yaml`). +PolicyGenerator CRs under `telco-core/configuration/` do not enumerate individual +MachineConfig files. + ## SiteConfig CRs The example-standard.yaml file contains a SiteConfig CR which defines the @@ -30,22 +47,36 @@ SiteConfig defines: ## extra-manifests -The CRs in extra-manifests are exact copies of some CRs from the -../configuration tree. These CRs will be applied during installation to -accelerate the time to cluster-ready. +Reference `MachineConfig` and `MachineConfigPool` CRs for Telco Core live **only** +in this directory. They are applied during installation (for example via +`ClusterInstance.spec.extraManifestsRefs`) and are the single copy in git; the +same content is validated at day-N using the Hub **extra-manifests** policy +(see `telco-hub/configuration/reference-crs/required/gitops/ztp-policies/extra-manifests-policy.yaml`). + +The cluster-compare reference under `../configuration/reference-crs-kube-compare/` +is kept aligned with these files via `make compare_extra_manifests` in `../configuration` +(part of `make check`). + +Reference `MachineConfig` files in this directory: + +- `control-plane-load-kernel-modules.yaml` +- `worker-load-kernel-modules.yaml` +- `mount_namespace_config_master.yaml` +- `mount_namespace_config_worker.yaml` +- `kdump-master.yaml` +- `kdump-worker.yaml` +- `mc_rootless_pods_selinux.yaml` +- `sctp_module_mc.yaml` +- `mcp-worker-1.yaml`, `mcp-worker-2.yaml`, `mcp-worker-3.yaml` (`MachineConfigPool`) + +The same three MCP files are duplicated under +`../configuration/reference-crs/custom-manifests/` for PolicyGenerator paths +(the plugin cannot reference `../install/`). `compare.sh --check-extra-manifests` +keeps install and custom-manifests copies in sync. ## custom-manifests -These CRs are an additional set of CRs which you want to apply to the cluster -during installation. The CRs here are treated in the same way as the -extra-manifests directory but are separated to make it easier to update the set -of reference manifests when new versions are released. - -The example manifests included here define two Machine Config Pools for the -cluster which bind nodes based on the node-role.kubernetes.io label. The -examples here also set the Machine Config Pools to `paused: true` and -`maxUnavailable: 100%`. This results in a significant improvement in install -times by allowing all worker nodes to update simultaneously (before any workload -is applied). The assumption is that post-installation the Machine Config Pools -will be set to `paused: false` and when nodes are ready the MCP set to a -reasonable `maxUnavailable: ` for your use case. +Optional additional manifests you maintain locally (see `README.md` in this +directory). Add a `ConfigMap` reference under `ClusterInstance.spec.extraManifestsRefs` +when you use this directory. The example `ClusterInstance` only references the +reference `extra-manifests-configmap` by default. diff --git a/telco-core/install/custom-manifests/README.md b/telco-core/install/custom-manifests/README.md new file mode 100644 index 000000000..21dd72d09 --- /dev/null +++ b/telco-core/install/custom-manifests/README.md @@ -0,0 +1,9 @@ +# Custom installation manifests + +Place additional manifests you want applied at cluster install time here and +include a `ConfigMap` (for example `custom-manifests-configmap`) in your +`ClusterInstance.spec.extraManifestsRefs`. + +The reference **MachineConfigPool** examples (`mcp-worker-1.yaml` through +`mcp-worker-3.yaml`) now live under `../extra-manifests/` together with the +other reference `MachineConfig` CRs so there is a single copy in git. diff --git a/telco-core/install/example-standard-clusterinstance.yaml b/telco-core/install/example-standard-clusterinstance.yaml index 5cd9037cb..670aed454 100644 --- a/telco-core/install/example-standard-clusterinstance.yaml +++ b/telco-core/install/example-standard-clusterinstance.yaml @@ -25,7 +25,6 @@ spec: # Reference configuration extra manifests extraManifestsRefs: - name: extra-manifests-configmap - - name: custom-manifests-configmap extraLabels: ManagedCluster: common: "core" diff --git a/telco-core/install/extra-manifests/mcp-worker-1.yaml b/telco-core/install/extra-manifests/mcp-worker-1.yaml new file mode 100644 index 000000000..f590e53fb --- /dev/null +++ b/telco-core/install/extra-manifests/mcp-worker-1.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + name: worker-1 + labels: + machineconfiguration.openshift.io/role: worker-1 +spec: + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: [ worker, worker-1 ] + paused: true + maxUnavailable: 100% + nodeSelector: + matchLabels: + node-role.kubernetes.io/worker-1: "" diff --git a/telco-core/install/extra-manifests/mcp-worker-2.yaml b/telco-core/install/extra-manifests/mcp-worker-2.yaml new file mode 100644 index 000000000..39aa39b4e --- /dev/null +++ b/telco-core/install/extra-manifests/mcp-worker-2.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + name: worker-2 + labels: + machineconfiguration.openshift.io/role: worker-2 +spec: + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: [ worker, worker-2 ] + paused: true + maxUnavailable: 100% + nodeSelector: + matchLabels: + node-role.kubernetes.io/worker-2: "" diff --git a/telco-core/install/extra-manifests/mcp-worker-3.yaml b/telco-core/install/extra-manifests/mcp-worker-3.yaml new file mode 100644 index 000000000..758dc6fb3 --- /dev/null +++ b/telco-core/install/extra-manifests/mcp-worker-3.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfigPool +metadata: + name: worker-3 + labels: + machineconfiguration.openshift.io/role: worker-3 +spec: + machineConfigSelector: + matchExpressions: + - key: machineconfiguration.openshift.io/role + operator: In + values: [ worker, worker-3 ] + paused: true + maxUnavailable: 100% + nodeSelector: + matchLabels: + node-role.kubernetes.io/worker-3: "" diff --git a/telco-core/install/extra-manifests/sctp_module_mc.yaml b/telco-core/install/extra-manifests/sctp_module_mc.yaml index 17b1e0b47..71e26380b 100644 --- a/telco-core/install/extra-manifests/sctp_module_mc.yaml +++ b/telco-core/install/extra-manifests/sctp_module_mc.yaml @@ -4,23 +4,22 @@ apiVersion: machineconfiguration.openshift.io/v1 kind: MachineConfig metadata: + name: load-sctp-module labels: machineconfiguration.openshift.io/role: worker - name: load-sctp-module spec: config: ignition: - version: 2.2.0 + version: 3.2.0 storage: files: - - contents: - source: data:, - verification: {} - filesystem: root + - path: /etc/modprobe.d/sctp-blacklist.conf mode: 420 - path: /etc/modprobe.d/sctp-blacklist.conf - - contents: - source: data:text/plain;charset=utf-8;base64,c2N0cA== - filesystem: root + overwrite: true + contents: + source: data:, + - path: /etc/modules-load.d/sctp-load.conf mode: 420 - path: /etc/modules-load.d/sctp-load.conf + overwrite: true + contents: + source: data:,sctp diff --git a/telco-core/install/kustomization.yaml b/telco-core/install/kustomization.yaml new file mode 100644 index 000000000..9cf62b8e7 --- /dev/null +++ b/telco-core/install/kustomization.yaml @@ -0,0 +1,29 @@ +--- +# Example: build the install-time extra-manifests ConfigMap for ClusterInstance. +# Apply with: kubectl apply -k telco-core/install/ +# The ClusterInstance in example-standard-clusterinstance.yaml references +# extraManifestsRefs: extra-manifests-configmap (see spec.extraManifestsRefs). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - example-standard-clusterinstance.yaml + +configMapGenerator: + - name: extra-manifests-configmap + namespace: example-standard + files: + - extra-manifests/control-plane-load-kernel-modules.yaml + - extra-manifests/worker-load-kernel-modules.yaml + - extra-manifests/mount_namespace_config_master.yaml + - extra-manifests/mount_namespace_config_worker.yaml + - extra-manifests/kdump-master.yaml + - extra-manifests/kdump-worker.yaml + - extra-manifests/mc_rootless_pods_selinux.yaml + - extra-manifests/sctp_module_mc.yaml + - extra-manifests/mcp-worker-1.yaml + - extra-manifests/mcp-worker-2.yaml + - extra-manifests/mcp-worker-3.yaml + +generatorOptions: + disableNameSuffixHash: true From 57c42ae4308ec421ec4d64e6879e29fecd14acd6 Mon Sep 17 00:00:00 2001 From: Keerthana Chennagoni Date: Tue, 23 Jun 2026 13:31:51 -0400 Subject: [PATCH 2/2] CNF-22194 Core: fix compare.sh for templated kube-compare MCs --- telco-core/configuration/compare.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/telco-core/configuration/compare.sh b/telco-core/configuration/compare.sh index e2f727f48..b0293af7f 100755 --- a/telco-core/configuration/compare.sh +++ b/telco-core/configuration/compare.sh @@ -128,11 +128,28 @@ compare_install_extra_manifests() { "kdump-worker.yaml:optional/other/kdump-worker.yaml" "mc_rootless_pods_selinux.yaml:optional/networking/multus/tap_cni/mc_rootless_pods_selinux.yaml" ) - local pair inst ref + local pair inst ref refpath for pair in "${pairs[@]}"; do inst="${pair%%:*}" ref="${pair##*:}" - if ! diff -u "${install}/${inst}" "${kubecmp}/${ref}"; then + refpath="${kubecmp}/${ref}" + # kube-compare uses Go templates for some MCs; install holds rendered reference YAML. + if grep -q '{{' "${refpath}"; then + if ! grep -q 'validateBase64List' "${refpath}"; then + echo "ERROR: expected validateBase64List templating in kube-compare ${ref}" >&2 + fail=1 + fi + if ! grep -q 'version: 3.2.0' "${install}/${inst}"; then + echo "ERROR: ${install}/${inst} must use ignition version 3.2.0" >&2 + fail=1 + fi + if ! grep -q 'path: /etc/modules-load.d/kernel-load.conf' "${install}/${inst}"; then + echo "ERROR: ${install}/${inst} must configure kernel-load.conf" >&2 + fail=1 + fi + continue + fi + if ! diff -u "${install}/${inst}" "${refpath}"; then echo "ERROR: install/extra-manifests/${inst} differs from kube-compare ${ref}" >&2 fail=1 fi