From 67dcab07f7bbfbd67f118307a23ce0632fd8ad39 Mon Sep 17 00:00:00 2001 From: Jim Ramsay Date: Fri, 15 May 2026 13:08:19 -0400 Subject: [PATCH] telco-ran: Pre-create /run/netns bindmount so it propagates to the container-mount-namespace This closes a race between running `ip netns add` inside the container-mount-namespace namespace and running it outside of the container-mount-namespace namespace. The cause of the race is that the /run/netns/ bindmount created by `ip netns add` is created with rshared properties, so the external call would shadow the internal call any time the internal call occurs first. Forcing the call to happen on the outside ensures any subsequent internal calls will simply reuse the existing bindmount (as intended). Signed-off-by: Jim Ramsay --- .../container-mount-namespace.service | 11 +++++++++++ ...01-container-mount-ns-and-kubelet-conf-master.yaml | 11 +++++++++++ ...01-container-mount-ns-and-kubelet-conf-worker.yaml | 11 +++++++++++ ...01-container-mount-ns-and-kubelet-conf-master.yaml | 11 +++++++++++ ...01-container-mount-ns-and-kubelet-conf-worker.yaml | 11 +++++++++++ 5 files changed, 55 insertions(+) diff --git a/telco-ran/configuration/extra-manifests-builder/01-container-mount-ns-and-kubelet-conf/container-mount-namespace.service b/telco-ran/configuration/extra-manifests-builder/01-container-mount-ns-and-kubelet-conf/container-mount-namespace.service index ed0876500..0072392bb 100644 --- a/telco-ran/configuration/extra-manifests-builder/01-container-mount-ns-and-kubelet-conf/container-mount-namespace.service +++ b/telco-ran/configuration/extra-manifests-builder/01-container-mount-ns-and-kubelet-conf/container-mount-namespace.service @@ -11,3 +11,14 @@ ExecStartPre=bash -c "findmnt ${RUNTIME_DIRECTORY} || mount --make-unbindable -- ExecStartPre=touch ${BIND_POINT} ExecStart=unshare --mount=${BIND_POINT} --propagation slave mount --make-rshared / ExecStop=umount -R ${RUNTIME_DIRECTORY} + +# Initialize /run/netns as a shared mount point to prevent shadowing race condition +# This must happen before CRI-O starts to ensure all namespace bind mounts have +# the correct parent mount. Without this, 'ip netns add' (called later by network +# components) would create the mount point and shadow any earlier namespace mounts +# created by CRI-O's pinns, causing pods to fail with 'setns: Invalid argument'. +# Creating and deleting a temporary netns retains the side-effect of creating the +# /run/netns mountpoint (but base exit code solely on whether the mountpoint is +# present) +# Reference: OCPBUGS-83562 +ExecStartPost=bash -c "ip netns add kubensmnt && ip netns delete kubensmnt || mountpoint -q /run/netns" diff --git a/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-master.yaml b/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-master.yaml index 88f2ad218..b4143c163 100644 --- a/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-master.yaml +++ b/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-master.yaml @@ -36,6 +36,17 @@ spec: ExecStartPre=touch ${BIND_POINT} ExecStart=unshare --mount=${BIND_POINT} --propagation slave mount --make-rshared / ExecStop=umount -R ${RUNTIME_DIRECTORY} + + # Initialize /run/netns as a shared mount point to prevent shadowing race condition + # This must happen before CRI-O starts to ensure all namespace bind mounts have + # the correct parent mount. Without this, 'ip netns add' (called later by network + # components) would create the mount point and shadow any earlier namespace mounts + # created by CRI-O's pinns, causing pods to fail with 'setns: Invalid argument'. + # Creating and deleting a temporary netns retains the side-effect of creating the + # /run/netns mountpoint (but base exit code solely on whether the mountpoint is + # present) + # Reference: OCPBUGS-83562 + ExecStartPost=bash -c "ip netns add kubensmnt && ip netns delete kubensmnt || mountpoint -q /run/netns" name: container-mount-namespace.service - dropins: - contents: | diff --git a/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-worker.yaml b/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-worker.yaml index 4697584be..9ec0e0c8b 100644 --- a/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-worker.yaml +++ b/telco-ran/configuration/kube-compare-reference/machine-config/kubelet-configuration-and-container-mount-hiding/01-container-mount-ns-and-kubelet-conf-worker.yaml @@ -36,6 +36,17 @@ spec: ExecStartPre=touch ${BIND_POINT} ExecStart=unshare --mount=${BIND_POINT} --propagation slave mount --make-rshared / ExecStop=umount -R ${RUNTIME_DIRECTORY} + + # Initialize /run/netns as a shared mount point to prevent shadowing race condition + # This must happen before CRI-O starts to ensure all namespace bind mounts have + # the correct parent mount. Without this, 'ip netns add' (called later by network + # components) would create the mount point and shadow any earlier namespace mounts + # created by CRI-O's pinns, causing pods to fail with 'setns: Invalid argument'. + # Creating and deleting a temporary netns retains the side-effect of creating the + # /run/netns mountpoint (but base exit code solely on whether the mountpoint is + # present) + # Reference: OCPBUGS-83562 + ExecStartPost=bash -c "ip netns add kubensmnt && ip netns delete kubensmnt || mountpoint -q /run/netns" name: container-mount-namespace.service - dropins: - contents: | diff --git a/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-master.yaml b/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-master.yaml index 88f2ad218..b4143c163 100644 --- a/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-master.yaml +++ b/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-master.yaml @@ -36,6 +36,17 @@ spec: ExecStartPre=touch ${BIND_POINT} ExecStart=unshare --mount=${BIND_POINT} --propagation slave mount --make-rshared / ExecStop=umount -R ${RUNTIME_DIRECTORY} + + # Initialize /run/netns as a shared mount point to prevent shadowing race condition + # This must happen before CRI-O starts to ensure all namespace bind mounts have + # the correct parent mount. Without this, 'ip netns add' (called later by network + # components) would create the mount point and shadow any earlier namespace mounts + # created by CRI-O's pinns, causing pods to fail with 'setns: Invalid argument'. + # Creating and deleting a temporary netns retains the side-effect of creating the + # /run/netns mountpoint (but base exit code solely on whether the mountpoint is + # present) + # Reference: OCPBUGS-83562 + ExecStartPost=bash -c "ip netns add kubensmnt && ip netns delete kubensmnt || mountpoint -q /run/netns" name: container-mount-namespace.service - dropins: - contents: | diff --git a/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-worker.yaml b/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-worker.yaml index 4697584be..9ec0e0c8b 100644 --- a/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-worker.yaml +++ b/telco-ran/configuration/source-crs/extra-manifest/01-container-mount-ns-and-kubelet-conf-worker.yaml @@ -36,6 +36,17 @@ spec: ExecStartPre=touch ${BIND_POINT} ExecStart=unshare --mount=${BIND_POINT} --propagation slave mount --make-rshared / ExecStop=umount -R ${RUNTIME_DIRECTORY} + + # Initialize /run/netns as a shared mount point to prevent shadowing race condition + # This must happen before CRI-O starts to ensure all namespace bind mounts have + # the correct parent mount. Without this, 'ip netns add' (called later by network + # components) would create the mount point and shadow any earlier namespace mounts + # created by CRI-O's pinns, causing pods to fail with 'setns: Invalid argument'. + # Creating and deleting a temporary netns retains the side-effect of creating the + # /run/netns mountpoint (but base exit code solely on whether the mountpoint is + # present) + # Reference: OCPBUGS-83562 + ExecStartPost=bash -c "ip netns add kubensmnt && ip netns delete kubensmnt || mountpoint -q /run/netns" name: container-mount-namespace.service - dropins: - contents: |