osd: load latest cephx key during osd pod init

Create a new OSD pod init container that loads the latest CephX key for
the OSD. The init container must be run after OSD activation, due to
keyfile reset behavior of ceph-bluestore-tool, used during activation.

The primary purpose of this work is to allow OSDs to start and run
correctly after key rotation, whether done manually by users or
automatically by future Rook work.

This work makes redundant prior code changes done in rook#14826. The CI test
added in the prior work is kept to ensure no regression.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>

Author: BlaineEXE

pkg/operator/ceph/cluster/osd/spec.go

@@ -111,39 +111,6 @@ KEYRING_FILE="$OSD_DATA_DIR"/keyring
 CV_MODE=%s
 DEVICE="$%s"
 
-# In rare cases keyring file created with prepare-osd but did not
-# being stored in ceph auth system therefore we need to import it
-# from keyring file instead of creating new one
-if ! ceph -n client.admin auth get osd."$OSD_ID" -k /etc/ceph/admin-keyring-store/keyring; then
-    if [ -f "$KEYRING_FILE" ]; then
-        # import keyring from existing file
-        TMP_DIR=$(mktemp -d)
-
-        python3 -c "
-import configparser
-
-config = configparser.ConfigParser()
-config.read('$KEYRING_FILE')
-
-if not config.has_section('osd.$OSD_ID'):
-    exit()
-
-config['osd.$OSD_ID'] = {'key': config['osd.$OSD_ID']['key'], 'caps mon': '\"allow profile osd\"', 'caps mgr': '\"allow profile osd\"', 'caps osd': '\"allow *\"'}
-
-with open('$TMP_DIR/keyring', 'w') as configfile:
-    config.write(configfile)
-"
-
-        cat "$TMP_DIR"/keyring
-        ceph -n client.admin auth import -i "$TMP_DIR"/keyring -k /etc/ceph/admin-keyring-store/keyring
-
-        rm --recursive --force "$TMP_DIR"
-    else
-        # create new keyring if no keyring file found
-        ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
-    fi
-fi
-
 # active the osd with ceph-volume
 if [[ "$CV_MODE" == "lvm" ]]; then
 	TMP_DIR=$(mktemp -d)
@@ -651,6 +618,13 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
 		initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd))
 	}
 
+	// key update init container must go after activate
+	adminKeyVol, cephxContainer := c.getCephxKeyUpdateInitContainer(osdID, osdProps)
+	if !volumeExistsWithName(volumes, adminKeyVol.Name) {
+		volumes = append(volumes, adminKeyVol)
+	}
+	initContainers = append(initContainers, cephxContainer)
+
 	// Doing a chown in a post start lifecycle hook does not reliably complete before the OSD
 	// process starts, which can cause the pod to fail without the lifecycle hook's chown command
 	// completing. It can take an arbitrarily long time for a pod restart to successfully chown the
@@ -1430,6 +1404,83 @@ func (c *Cluster) getEncryptedStatusPVCInitContainer(mountPath string, osdProps
 	}
 }
 
+// init container to get (or create, if needed) the most up-to-date keyring for the given OSD and
+// update the OSD config directory keyring file. this allows OSD cephx keys to be rotated without
+// causing OSD pod crash loops.
+//
+// this must come after OSD activate init container because 'ceph-bluestore-tool' (and consequently,
+// 'ceph-volume activate') pulls the keyring from the OSD disk's bluestore data and overwrites the
+// config dir's keyring file, even when the on-disk key is not the most up-to-date.
+func (c *Cluster) getCephxKeyUpdateInitContainer(osdID string, osdProps osdProperties) (v1.Volume, v1.Container) {
+	volMounts := []v1.VolumeMount{
+		{Name: k8sutil.ConfigOverrideName, ReadOnly: true, MountPath: opconfig.EtcCephDir},
+	}
+
+	adminKeyringVol, adminKeyringVolMount := cephkey.Volume().Admin(), cephkey.VolumeMount().Admin()
+	volMounts = append(volMounts, adminKeyringVolMount)
+
+	// OSD config dir is at /var/lib/ceph/osd/ceph-<id>
+	osdConfigMountPath := activateOSDMountPath + osdID
+
+	if osdProps.onPVC() {
+		// assumption: bridge vol will already be added to pod spec for PVC OSDs
+		volMounts = append(volMounts, getPvcOSDBridgeMountActivate(osdConfigMountPath, osdProps.pvc.ClaimName))
+	} else {
+		// assumption: activate OSD volume will already be added to pod spec for non-PVC OSDs at the
+		// correct location for mounting to /var/lib/ceph/osd/ceph-<id>
+		volMounts = append(volMounts,
+			v1.VolumeMount{Name: activateOSDVolumeName, MountPath: osdConfigMountPath},
+		)
+	}
+
+	keyUpdateScript := `
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+OSD_ID="` + osdID + `"
+KEYRING_FILE=/var/lib/ceph/osd/ceph-"${OSD_ID}"/keyring
+
+ceph --name client.admin auth get-or-create osd."${OSD_ID}" \
+  mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' \
+  --keyring /etc/ceph/admin-keyring-store/keyring > "$KEYRING_FILE"
+`
+
+	envVars := []v1.EnvVar{
+		{
+			Name: "ROOK_CEPH_MON_HOST",
+			ValueFrom: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: "rook-ceph-config",
+					},
+					Key: "mon_host",
+				},
+			},
+		},
+		{Name: "CEPH_ARGS", Value: "--mon-host=$(ROOK_CEPH_MON_HOST)"},
+	}
+
+	container := v1.Container{
+		Command: []string{
+			"/bin/bash",
+			"-c",
+			keyUpdateScript,
+		},
+		Name:            "cephx-keyring-update",
+		Image:           c.spec.CephVersion.Image,
+		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
+		VolumeMounts:    volMounts,
+		SecurityContext: controller.PrivilegedContext(true),
+		Env:             envVars,
+		EnvFrom:         getEnvFromSources(),
+		Resources:       osdProps.resources,
+	}
+
+	return adminKeyringVol, container
+}
+
 func (c *Cluster) getOSDContainerPorts() []v1.ContainerPort {
 	var ports []v1.ContainerPort
 	if c.spec.RequireMsgr2() {
@@ -1495,3 +1546,12 @@ func getOSDCmd(cmd []string, interval int) []string {
 	}
 	return cmd
 }
+
+func volumeExistsWithName(vols []v1.Volume, name string) bool {
+	for _, v := range vols {
+		if v.Name == name {
+			return true
+		}
+	}
+	return false
+}