fix(control-plane-operator): validate cloud config hash in ignition server

- Add cloudConfigHash parameter to IgnitionProvider.GetPayload interface
  (appended after osStream to preserve existing parameter positions)
- Gate ignition payload generation on cloud config hash match in
  LocalIgnitionProvider, returning an error when the ConfigMap is stale
- Invalidate cached payloads when cloud config hash changes by storing
  the hash in CacheValue and comparing on reconciliation
- Report CloudConfigPending reason on token secret when hash validation
  fails so NodePool controller can observe convergence state

Signed-off-by: Todd Wolff <twolff@redhat.com>
Commit-Message-Assisted-by: Claude (via Claude Code)

Author: twolff-gh

ignition-server/cmd/run_local_ignitionprovider.go

@@ -112,7 +112,7 @@ func (o *RunLocalIgnitionProviderOptions) Run(ctx context.Context) error {
 	}
 
 	osStream := string(token.Data[controllers.TokenSecretOSStreamKey])
-	payload, err := p.GetPayload(ctx, o.Image, config.String(), "", "", "", osStream)
+	payload, err := p.GetPayload(ctx, o.Image, config.String(), "", "", "", osStream, "")
 	if err != nil {
 		return err
 	}

ignition-server/controllers/cache.go

@@ -32,8 +32,9 @@ type ExpiringCache struct {
 }
 
 type CacheValue struct {
-	Payload    []byte
-	SecretName string
+	Payload         []byte
+	SecretName      string
+	CloudConfigHash string
 }
 
 type entry struct {

ignition-server/controllers/local_ignitionprovider.go

@@ -300,8 +300,10 @@ func (p *LocalIgnitionProvider) extractImageReferences(ctx context.Context, rele
 	return releaseImage, nil
 }
 
-// For Azure and OpenStack, extract the cloud provider config file as MCO input
-func (p *LocalIgnitionProvider) writeCloudProviderConfig(ctx context.Context, mcoDir string) error {
+// For Azure and OpenStack, extract the cloud provider config file as MCO input.
+// When cloudConfigHash is non-empty, the ConfigMap content is verified against the hash
+// before writing to ensure the ignition server does not serve stale cloud config.
+func (p *LocalIgnitionProvider) writeCloudProviderConfig(ctx context.Context, mcoDir, cloudConfigHash string) error {
 	if p.CloudProvider != hyperv1.AzurePlatform && p.CloudProvider != hyperv1.OpenStackPlatform {
 		return nil
 	}
@@ -316,6 +318,13 @@ func (p *LocalIgnitionProvider) writeCloudProviderConfig(ctx context.Context, mc
 			return fmt.Errorf("failed to get cloud provider configmap: %w", err)
 		}
 	}
+	if cloudConfigHash != "" {
+		actualHash := util.HashConfigMapData(cloudConfigMap.Data)
+		if actualHash != cloudConfigHash {
+			return fmt.Errorf("cloud config %s/%s hash mismatch (expected %s, got %s), waiting for update",
+				cloudConfigMap.Namespace, cloudConfigMap.Name, cloudConfigHash, actualHash)
+		}
+	}
 	cloudConfYaml, err := yaml.Marshal(cloudConfigMap)
 	if err != nil {
 		return fmt.Errorf("failed to marshal cloud config: %w", err)
@@ -607,7 +616,7 @@ func (p *LocalIgnitionProvider) runMCSAndFetchPayload(ctx context.Context, dirs
 	return payload, err
 }
 
-func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, customConfig, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, osStream string) ([]byte, error) {
+func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, customConfig, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, osStream, cloudConfigHash string) ([]byte, error) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
 
@@ -682,7 +691,7 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, cu
 		return nil, fmt.Errorf("failed to extract image-references from image: %w", err)
 	}
 
-	if err := p.writeCloudProviderConfig(ctx, dirs.mcoDir); err != nil {
+	if err := p.writeCloudProviderConfig(ctx, dirs.mcoDir, cloudConfigHash); err != nil {
 		return nil, err
 	}
 

ignition-server/controllers/local_ignitionprovider_test.go

@@ -16,6 +16,7 @@ import (
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
+	"github.com/openshift/hypershift/support/util"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -1105,11 +1106,12 @@ func TestWriteCloudProviderConfig(t *testing.T) {
 	t.Parallel()
 
 	tests := []struct {
-		name          string
-		cloudProvider hyperv1.PlatformType
-		objects       []client.Object
-		expectFile    bool
-		expectError   string
+		name            string
+		cloudProvider   hyperv1.PlatformType
+		objects         []client.Object
+		cloudConfigHash string
+		expectFile      bool
+		expectError     string
 	}{
 		{
 			name:          "When cloud provider is AWS, it should return nil without writing any file",
@@ -1149,6 +1151,30 @@ func TestWriteCloudProviderConfig(t *testing.T) {
 			objects:       []client.Object{},
 			expectError:   "failed to get cloud provider configmap",
 		},
+		{
+			name:          "When cloud provider is Azure and hash matches, it should write cloud config file",
+			cloudProvider: hyperv1.AzurePlatform,
+			objects: []client.Object{
+				&corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: "azure-cloud-config", Namespace: "test-ns"},
+					Data:       map[string]string{"cloud.conf": "azure-config-data"},
+				},
+			},
+			cloudConfigHash: util.HashConfigMapData(map[string]string{"cloud.conf": "azure-config-data"}),
+			expectFile:      true,
+		},
+		{
+			name:          "When cloud provider is Azure and hash does not match, it should return an error",
+			cloudProvider: hyperv1.AzurePlatform,
+			objects: []client.Object{
+				&corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{Name: "azure-cloud-config", Namespace: "test-ns"},
+					Data:       map[string]string{"cloud.conf": "azure-config-data"},
+				},
+			},
+			cloudConfigHash: "wrong-hash",
+			expectError:     "hash mismatch",
+		},
 	}
 
 	for _, tt := range tests {
@@ -1165,7 +1191,7 @@ func TestWriteCloudProviderConfig(t *testing.T) {
 				CloudProvider: tt.cloudProvider,
 			}
 
-			err := provider.writeCloudProviderConfig(t.Context(), mcoDir)
+			err := provider.writeCloudProviderConfig(t.Context(), mcoDir, tt.cloudConfigHash)
 			if tt.expectError != "" {
 				g.Expect(err).To(HaveOccurred())
 				g.Expect(err.Error()).To(ContainSubstring(tt.expectError))

ignition-server/controllers/tokensecret_controller.go

@@ -35,7 +35,9 @@ const (
 	TokenSecretPullSecretHashKey            = "pull-secret-hash"
 	TokenSecretHCConfigurationHashKey       = "hc-configuration-hash"
 	TokenSecretAdditionalTrustBundleHashKey = "additional-trust-bundle-hash"
+	TokenSecretCloudConfigHashKey           = "cloud-config-hash"
 	InvalidConfigReason                     = "InvalidConfig"
+	CloudConfigPendingReason                = "CloudConfigPending"
 	TokenSecretReasonKey                    = "reason"
 	// TokenSecretOSStreamKey is intentionally duplicated from nodepool/token.go
 	// to avoid a dependency from ignition-server → hypershift-operator.
@@ -88,7 +90,7 @@ func NewPayloadStore() *ExpiringCache {
 type IgnitionProvider interface {
 	// GetPayload returns the ignition payload content for
 	// the provided release image and a config string containing 0..N MachineConfig yaml definitions.
-	GetPayload(ctx context.Context, payloadImage, config, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, osStream string) ([]byte, error)
+	GetPayload(ctx context.Context, payloadImage, config, pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, osStream, cloudConfigHash string) ([]byte, error)
 }
 
 // TokenSecretReconciler watches token Secrets
@@ -231,25 +233,35 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	token := string(tokenSecret.Data[TokenSecretTokenKey])
+	cloudConfigHash := string(tokenSecret.Data[TokenSecretCloudConfigHashKey])
 	if value, ok := r.PayloadStore.Get(token); ok {
-		log.Info("Payload found in cache")
-
-		if tokenNeedRotation(timeLived) {
-			log.Info("Rotating token ID")
-			if err := r.rotateToken(ctx, tokenSecret, value, now); err != nil {
-				return ctrl.Result{}, err
+		if value.CloudConfigHash != cloudConfigHash {
+			log.Info("Cloud config hash changed, invalidating cached payload")
+			r.PayloadStore.Delete(token)
+		} else {
+			log.Info("Payload found in cache")
+
+			if tokenNeedRotation(timeLived) {
+				log.Info("Rotating token ID")
+				if err := r.rotateToken(ctx, tokenSecret, value, now); err != nil {
+					return ctrl.Result{}, err
+				}
+				TokenRotationTotal.Inc()
 			}
-			TokenRotationTotal.Inc()
+			return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil
 		}
-		return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil
 	}
 
 	// If something else rotated the token (e.g. running in HA), we fall back to set the cache value from the old one.
 	oldToken, ok := tokenSecret.Data[TokenSecretOldTokenKey]
 	if ok {
 		if value, ok := r.PayloadStore.Get(string(oldToken)); ok {
-			r.PayloadStore.Set(token, value)
-			return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil
+			if value.CloudConfigHash == cloudConfigHash {
+				r.PayloadStore.Set(token, value)
+				return ctrl.Result{RequeueAfter: ttl/2 - durationDeref(timeLived)}, nil
+			}
+			log.Info("Cloud config hash changed, invalidating old token cached payload")
+			r.PayloadStore.Delete(string(oldToken))
 		}
 	}
 
@@ -279,7 +291,7 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	osStream := string(tokenSecret.Data[TokenSecretOSStreamKey])
 	payload, err := func() ([]byte, error) {
 		start := time.Now()
-		payload, err := r.IgnitionProvider.GetPayload(ctx, releaseImage, config.String(), pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, osStream)
+		payload, err := r.IgnitionProvider.GetPayload(ctx, releaseImage, config.String(), pullSecretHash, additionalTrustBundleHash, hcConfigurationHash, osStream, cloudConfigHash)
 		if err != nil {
 			return nil, fmt.Errorf("error getting ignition payload: %w", err)
 		}
@@ -292,12 +304,16 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		// This patch could flood the API server, so we should only do it when the reason or message is different from the current one.
 		// More info here: https://issues.redhat.com/browse/OCPBUGS-42320.
 		errWithFullMsg := fmt.Errorf("failed to generate payload: %w", err)
-		if hasSameReasonAndMessage(tokenSecret, InvalidConfigReason, errWithFullMsg) {
+		reason := InvalidConfigReason
+		if strings.Contains(err.Error(), "hash mismatch") {
+			reason = CloudConfigPendingReason
+		}
+		if hasSameReasonAndMessage(tokenSecret, reason, errWithFullMsg) {
 			return ctrl.Result{}, errWithFullMsg
 		}
 
 		patch := tokenSecret.DeepCopy()
-		patch.Data[TokenSecretReasonKey] = []byte(InvalidConfigReason)
+		patch.Data[TokenSecretReasonKey] = []byte(reason)
 		patch.Data[TokenSecretMessageKey] = []byte(errWithFullMsg.Error())
 		if err := r.Client.Patch(ctx, patch, client.MergeFrom(tokenSecret)); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to patch tokenSecret with payload content: %w", err)
@@ -307,12 +323,13 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 
 	log.Info("IgnitionProvider generated payload")
-	r.PayloadStore.Set(token, CacheValue{Payload: payload, SecretName: tokenSecret.Name})
+	cacheValue := CacheValue{Payload: payload, SecretName: tokenSecret.Name, CloudConfigHash: cloudConfigHash}
+	r.PayloadStore.Set(token, cacheValue)
 	oldToken, ok = tokenSecret.Data[TokenSecretOldTokenKey]
 	if ok {
 		// If we got here and there's an old token e.g. ignition server pod was restarted, then we set it as well
 		// So Machines that were given that token right before the restart can succeed.
-		r.PayloadStore.Set(string(oldToken), CacheValue{Payload: payload, SecretName: tokenSecret.Name})
+		r.PayloadStore.Set(string(oldToken), cacheValue)
 	}
 
 	patch := tokenSecret.DeepCopy()