Merge pull request rook#16003 from BlaineEXE/osd-allow-manual-rotation

osd: load latest cephx key during osd pod init

Author: BlaineEXE

pkg/operator/ceph/cluster/osd/spec.go

@@ -111,39 +111,6 @@ KEYRING_FILE="$OSD_DATA_DIR"/keyring
 CV_MODE=%s
 DEVICE="$%s"
 
-# In rare cases keyring file created with prepare-osd but did not
-# being stored in ceph auth system therefore we need to import it
-# from keyring file instead of creating new one
-if ! ceph -n client.admin auth get osd."$OSD_ID" -k /etc/ceph/admin-keyring-store/keyring; then
-    if [ -f "$KEYRING_FILE" ]; then
-        # import keyring from existing file
-        TMP_DIR=$(mktemp -d)
-
-        python3 -c "
-import configparser
-
-config = configparser.ConfigParser()
-config.read('$KEYRING_FILE')
-
-if not config.has_section('osd.$OSD_ID'):
-    exit()
-
-config['osd.$OSD_ID'] = {'key': config['osd.$OSD_ID']['key'], 'caps mon': '\"allow profile osd\"', 'caps mgr': '\"allow profile osd\"', 'caps osd': '\"allow *\"'}
-
-with open('$TMP_DIR/keyring', 'w') as configfile:
-    config.write(configfile)
-"
-
-        cat "$TMP_DIR"/keyring
-        ceph -n client.admin auth import -i "$TMP_DIR"/keyring -k /etc/ceph/admin-keyring-store/keyring
-
-        rm --recursive --force "$TMP_DIR"
-    else
-        # create new keyring if no keyring file found
-        ceph -n client.admin auth get-or-create osd."$OSD_ID" mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
-    fi
-fi
-
 # active the osd with ceph-volume
 if [[ "$CV_MODE" == "lvm" ]]; then
 	TMP_DIR=$(mktemp -d)
@@ -651,6 +618,13 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
 		initContainers = append(initContainers, c.getExpandInitContainer(osdProps, c.spec.DataDirHostPath, c.clusterInfo.Namespace, osdID, *osd))
 	}
 
+	// key update init container must go after activate
+	adminKeyVol, cephxContainer := c.getCephxKeyUpdateInitContainer(osdID, osdProps)
+	if !volumeExistsWithName(volumes, adminKeyVol.Name) {
+		volumes = append(volumes, adminKeyVol)
+	}
+	initContainers = append(initContainers, cephxContainer)
+
 	// Doing a chown in a post start lifecycle hook does not reliably complete before the OSD
 	// process starts, which can cause the pod to fail without the lifecycle hook's chown command
 	// completing. It can take an arbitrarily long time for a pod restart to successfully chown the
@@ -1430,6 +1404,83 @@ func (c *Cluster) getEncryptedStatusPVCInitContainer(mountPath string, osdProps
 	}
 }
 
+// init container to get (or create, if needed) the most up-to-date keyring for the given OSD and
+// update the OSD config directory keyring file. this allows OSD cephx keys to be rotated without
+// causing OSD pod crash loops.
+//
+// this must come after OSD activate init container because 'ceph-bluestore-tool' (and consequently,
+// 'ceph-volume activate') pulls the keyring from the OSD disk's bluestore data and overwrites the
+// config dir's keyring file, even when the on-disk key is not the most up-to-date.
+func (c *Cluster) getCephxKeyUpdateInitContainer(osdID string, osdProps osdProperties) (v1.Volume, v1.Container) {
+	volMounts := []v1.VolumeMount{
+		{Name: k8sutil.ConfigOverrideName, ReadOnly: true, MountPath: opconfig.EtcCephDir},
+	}
+
+	adminKeyringVol, adminKeyringVolMount := cephkey.Volume().Admin(), cephkey.VolumeMount().Admin()
+	volMounts = append(volMounts, adminKeyringVolMount)
+
+	// OSD config dir is at /var/lib/ceph/osd/ceph-<id>
+	osdConfigMountPath := activateOSDMountPath + osdID
+
+	if osdProps.onPVC() {
+		// assumption: bridge vol will already be added to pod spec for PVC OSDs
+		volMounts = append(volMounts, getPvcOSDBridgeMountActivate(osdConfigMountPath, osdProps.pvc.ClaimName))
+	} else {
+		// assumption: activate OSD volume will already be added to pod spec for non-PVC OSDs at the
+		// correct location for mounting to /var/lib/ceph/osd/ceph-<id>
+		volMounts = append(volMounts,
+			v1.VolumeMount{Name: activateOSDVolumeName, MountPath: osdConfigMountPath},
+		)
+	}
+
+	keyUpdateScript := `
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+OSD_ID="` + osdID + `"
+KEYRING_FILE=/var/lib/ceph/osd/ceph-"${OSD_ID}"/keyring
+
+ceph --name client.admin auth get-or-create osd."${OSD_ID}" \
+  mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' \
+  --keyring /etc/ceph/admin-keyring-store/keyring > "$KEYRING_FILE"
+`
+
+	envVars := []v1.EnvVar{
+		{
+			Name: "ROOK_CEPH_MON_HOST",
+			ValueFrom: &v1.EnvVarSource{
+				SecretKeyRef: &v1.SecretKeySelector{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: "rook-ceph-config",
+					},
+					Key: "mon_host",
+				},
+			},
+		},
+		{Name: "CEPH_ARGS", Value: "--mon-host=$(ROOK_CEPH_MON_HOST)"},
+	}
+
+	container := v1.Container{
+		Command: []string{
+			"/bin/bash",
+			"-c",
+			keyUpdateScript,
+		},
+		Name:            "cephx-keyring-update",
+		Image:           c.spec.CephVersion.Image,
+		ImagePullPolicy: controller.GetContainerImagePullPolicy(c.spec.CephVersion.ImagePullPolicy),
+		VolumeMounts:    volMounts,
+		SecurityContext: controller.PrivilegedContext(true),
+		Env:             envVars,
+		EnvFrom:         getEnvFromSources(),
+		Resources:       osdProps.resources,
+	}
+
+	return adminKeyringVol, container
+}
+
 func (c *Cluster) getOSDContainerPorts() []v1.ContainerPort {
 	var ports []v1.ContainerPort
 	if c.spec.RequireMsgr2() {
@@ -1495,3 +1546,12 @@ func getOSDCmd(cmd []string, interval int) []string {
 	}
 	return cmd
 }
+
+func volumeExistsWithName(vols []v1.Volume, name string) bool {
+	for _, v := range vols {
+		if v.Name == name {
+			return true
+		}
+	}
+	return false
+}