fix(federation): bridge terminal referenced-data error hub→cell via annotation

The referenced-data resolver sets ReferencedDataReady=False/SourceNotFound on the
hub WorkloadDeployment, but Karmada statusAggregation only flows cell->hub, so the
cell Instance controller (which reads the cell WD copy) never saw the verdict and
the Instance stayed Ready=Unknown/Resolving instead of Ready=False/SourceNotFound.

Bridge it via an annotation (compute.datumapis.com/referenced-data-error), which
DOES propagate hub->cell as part of ObjectMeta. The resolver stamps it on terminal
errors (SourceNotFound/TooLarge/Unauthorized) and clears it on resolve; the cell
instance controller reads it from the owner WD and surfaces the terminal reason+
message, with a condition-based fallback preserved for single-cluster.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
(cherry picked from commit 3751f78)

Author: scotwells

api/v1alpha/annotations.go

@@ -25,4 +25,25 @@ const (
 	// gate. Its value is an RFC3339 timestamp. Used to compute gate-wait duration
 	// for the compute_referenced_data_gate_wait_seconds histogram.
 	ReferencedDataGateStartAnnotation = AnnotationNamespace + "/referenced-data-gate-start"
+
+	// ReferencedDataErrorAnnotation is stamped on a WorkloadDeployment by the
+	// ReferencedDataController when a terminal source error occurs (SourceNotFound,
+	// SourceUnauthorized, or SourceTooLarge). Its value is a JSON object with
+	// "reason" and "message" fields carrying the authoritative resolver verdict.
+	//
+	// Example value:
+	//   {"reason":"SourceNotFound","message":"ConfigMap \"app-config\" not found in namespace \"default\""}
+	//
+	// This annotation bridges the federation boundary: Karmada propagates
+	// metadata.annotations hub→cell alongside WorkloadDeployment objects, but
+	// status.conditions do not propagate in that direction. The cell
+	// InstanceReconciler reads this annotation from the cell WD copy (returned by
+	// fetchOwnerWorkloadDeployment) and promotes it to the Instance's
+	// ReferencedDataReady condition so the terminal error is visible at the Instance
+	// level without requiring a cross-plane condition read.
+	//
+	// The annotation is removed when the error resolves (companion materialises /
+	// ReferencedDataReady flips True), so the absence of the annotation means
+	// either no error or the error has cleared.
+	ReferencedDataErrorAnnotation = AnnotationNamespace + "/referenced-data-error"
 )

internal/controller/instance_controller.go

@@ -501,6 +501,14 @@ func (r *InstanceReconciler) reconcileReferencedDataCondition(
 		return referencedDataResult{}, 0, err
 	}
 
+	// When the resolver has determined a terminal source error (the source object
+	// is missing, unauthorized, or too large), the companion will never arrive.
+	// Promote the terminal error to the Instance condition so it is visible without
+	// a secondary fetch. Returns (changed, true) when a terminal error was found.
+	if changed, terminal := r.applyTerminalErrorFromWD(ctx, wd, instance); terminal {
+		return referencedDataResult{conditionChanged: changed}, 0, nil
+	}
+
 	// Annotation not yet present — the resolver hasn't finished. Signal Resolving
 	// and requeue so we re-check even without a new watch event.
 	annoRaw, hasAnno := wd.Annotations[computev1alpha.ExpectedReferencedDataAnnotation]
@@ -613,6 +621,75 @@ func (r *InstanceReconciler) setReferencedDataConditionWithTransition(
 	return changed
 }
 
+// isTerminalReferencedDataReason reports whether the given ReferencedData reason
+// is terminal — i.e., the companion will never arrive because the source object
+// is permanently unavailable, not just slow to propagate.
+func isTerminalReferencedDataReason(reason string) bool {
+	switch reason {
+	case computev1alpha.ReferencedDataReasonSourceNotFound,
+		computev1alpha.ReferencedDataReasonSourceUnauthorized,
+		computev1alpha.ReferencedDataReasonSourceTooLarge:
+		return true
+	}
+	return false
+}
+
+// applyTerminalErrorFromWD reads the terminal-error signal from the owner WD and,
+// when a valid terminal error is found, sets the Instance's ReferencedDataReady
+// condition. Returns (changed, true) when a terminal error was applied, or
+// (false, false) when there is no terminal error to apply.
+//
+// The signal is read from two places in priority order:
+//  1. ReferencedDataErrorAnnotation on the WD metadata — the primary path in
+//     federation, because Karmada propagates annotations hub→cell but does not
+//     propagate status.conditions in that direction.
+//  2. The WD's ReferencedDataReady status condition — the fallback for
+//     single-cluster deployments or during a controller version upgrade before
+//     the annotation was introduced.
+func (r *InstanceReconciler) applyTerminalErrorFromWD(
+	ctx context.Context,
+	wd *computev1alpha.WorkloadDeployment,
+	instance *computev1alpha.Instance,
+) (changed, terminal bool) {
+	// Path 1: annotation (federation-safe).
+	if termErrRaw := wd.Annotations[computev1alpha.ReferencedDataErrorAnnotation]; termErrRaw != "" {
+		termReason, termMessage, decodeErr := decodeTerminalErrorAnnotation(termErrRaw)
+		if decodeErr != nil {
+			log.FromContext(ctx).V(1).Info("malformed referenced-data-error annotation on WD; ignoring",
+				"workloadDeployment", wd.Name, "error", decodeErr)
+		} else if isTerminalReferencedDataReason(termReason) {
+			ch := r.setReferencedDataConditionWithTransition(instance, metav1.ConditionFalse,
+				termReason, termMessage)
+			return ch, true
+		}
+	}
+
+	// Path 2: status condition fallback (single-cluster).
+	wdCond := apimeta.FindStatusCondition(wd.Status.Conditions, computev1alpha.ReferencedDataReady)
+	if wdCond != nil && wdCond.Status == metav1.ConditionFalse && isTerminalReferencedDataReason(wdCond.Reason) {
+		ch := r.setReferencedDataConditionWithTransition(instance, metav1.ConditionFalse,
+			wdCond.Reason, wdCond.Message)
+		return ch, true
+	}
+
+	return false, false
+}
+
+// decodeTerminalErrorAnnotation parses the value of ReferencedDataErrorAnnotation
+// into (reason, message). Returns an error when the annotation is malformed.
+// Both files in the same package share this unexported helper through Go's
+// package-level visibility — no cross-file coupling issue.
+func decodeTerminalErrorAnnotation(raw string) (reason, message string, err error) {
+	var payload struct {
+		Reason  string `json:"reason"`
+		Message string `json:"message"`
+	}
+	if err := json.Unmarshal([]byte(raw), &payload); err != nil {
+		return "", "", fmt.Errorf("decode referenced-data-error annotation %q: %w", raw, err)
+	}
+	return payload.Reason, payload.Message, nil
+}
+
 // listPresentCompanionsByKindName returns a set keyed by kind-qualified tokens
 // ("Kind/name", e.g. "ConfigMap/app-config") for every companion ConfigMap and
 // Secret present in the given namespace (matched by ReferencedDataLabel). This

internal/controller/instance_referenced_data_test.go

@@ -564,6 +564,217 @@ func TestReferencedDataStaleConditionGuard(t *testing.T) {
 	}
 }
 
+// ─── Federation annotation bridge: terminal error propagation hub→cell ────────
+
+// makeWDWithTerminalError returns a WD carrying the ReferencedDataErrorAnnotation
+// as a cell WD copy would after Karmada propagation from the hub.
+func makeWDWithTerminalError(reason, message string) *computev1alpha.WorkloadDeployment {
+	raw, _ := encodeTerminalError(reason, message)
+	wd := &computev1alpha.WorkloadDeployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      refDataTestDeployment,
+			Namespace: refDataTestNamespace,
+			UID:       refDataTestWDUID,
+			Annotations: map[string]string{
+				computev1alpha.ReferencedDataErrorAnnotation: raw,
+			},
+		},
+	}
+	return wd
+}
+
+// TestFederated_TerminalAnnotation_SourceNotFound verifies that when the cell WD
+// copy carries the ReferencedDataErrorAnnotation with SourceNotFound, the Instance
+// gets Ready=False/SourceNotFound with the hub message rather than AwaitingPropagation.
+func TestFederated_TerminalAnnotation_SourceNotFound(t *testing.T) {
+	msg := `ConfigMap "app-config" not found in namespace "default"`
+	wd := makeWDWithTerminalError(computev1alpha.ReferencedDataReasonSourceNotFound, msg)
+	inst := makeInstanceWithRefDataGate()
+
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond, "ReferencedDataReady condition should be set")
+	assert.Equal(t, metav1.ConditionFalse, cond.Status)
+	assert.Equal(t, computev1alpha.ReferencedDataReasonSourceNotFound, cond.Reason,
+		"should use the hub resolver reason, not generic AwaitingPropagation")
+	assert.Equal(t, msg, cond.Message,
+		"should carry the hub resolver message verbatim")
+
+	// Gate must remain until the error is resolved (companion will never arrive).
+	hasGate := false
+	if updated.Spec.Controller != nil {
+		for _, g := range updated.Spec.Controller.SchedulingGates {
+			if g.Name == instancecontrol.ReferencedDataSchedulingGate.String() {
+				hasGate = true
+			}
+		}
+	}
+	assert.True(t, hasGate, "ReferencedData gate should remain when a terminal error is present")
+}
+
+// TestFederated_TerminalAnnotation_SourceTooLarge verifies SourceTooLarge propagates
+// via the annotation the same way as SourceNotFound.
+func TestFederated_TerminalAnnotation_SourceTooLarge(t *testing.T) {
+	msg := `ConfigMap "fat-config" in namespace "default" exceeds per-object size limit`
+	wd := makeWDWithTerminalError(computev1alpha.ReferencedDataReasonSourceTooLarge, msg)
+	inst := makeInstanceWithRefDataGate()
+
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond)
+	assert.Equal(t, metav1.ConditionFalse, cond.Status)
+	assert.Equal(t, computev1alpha.ReferencedDataReasonSourceTooLarge, cond.Reason)
+	assert.Equal(t, msg, cond.Message)
+}
+
+// TestFederated_TerminalAnnotation_SourceUnauthorized verifies SourceUnauthorized propagates
+// via the annotation.
+func TestFederated_TerminalAnnotation_SourceUnauthorized(t *testing.T) {
+	msg := `not authorized to read ConfigMap "secret-cfg" in namespace "default"`
+	wd := makeWDWithTerminalError(computev1alpha.ReferencedDataReasonSourceUnauthorized, msg)
+	inst := makeInstanceWithRefDataGate()
+
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond)
+	assert.Equal(t, metav1.ConditionFalse, cond.Status)
+	assert.Equal(t, computev1alpha.ReferencedDataReasonSourceUnauthorized, cond.Reason)
+	assert.Equal(t, msg, cond.Message)
+}
+
+// TestFederated_NoAnnotation_AnnotationAbsent verifies that when the WD has no
+// terminal-error annotation AND no expected-data annotation (resolver not done),
+// the Instance gets Resolving/Unknown — NOT SourceNotFound.
+func TestFederated_NoAnnotation_AnnotationAbsent(t *testing.T) {
+	// WD has neither annotation — Karmada propagated it before the resolver ran.
+	wd := makeWDForCell("") // no annotations
+	inst := makeInstanceWithRefDataGate()
+
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond)
+	assert.Equal(t, metav1.ConditionUnknown, cond.Status,
+		"should be Unknown/Resolving when no annotation is present")
+	assert.Equal(t, computev1alpha.ReferencedDataReasonResolving, cond.Reason)
+}
+
+// TestFederated_AnnotationPresent_CompanionsMissing verifies that when the WD has
+// the expected-data annotation (resolver succeeded, no terminal error) but companions
+// haven't arrived yet, the Instance gets AwaitingPropagation — not SourceNotFound.
+func TestFederated_AnnotationPresent_CompanionsMissing(t *testing.T) {
+	expected := []string{refDataTestCMToken}
+	annoVal, _ := json.Marshal(expected)
+	// WD has the expected annotation but NOT the terminal-error annotation.
+	wd := makeWDForCell(string(annoVal))
+	inst := makeInstanceWithRefDataGate()
+
+	// No companion ConfigMap present yet.
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond)
+	assert.Equal(t, metav1.ConditionFalse, cond.Status)
+	assert.Equal(t, computev1alpha.ReferencedDataReasonAwaitingPropagation, cond.Reason,
+		"should be AwaitingPropagation (not SourceNotFound) when resolver succeeded but companion is still in flight")
+	assert.Contains(t, cond.Message, refDataTestCMToken)
+}
+
+// TestFederated_AnnotationPresent_CompanionsPresent verifies that when the WD
+// has the expected-data annotation and all companions are present (healthy path),
+// the Instance advances to Ready=True.
+func TestFederated_AnnotationPresent_CompanionsPresent(t *testing.T) {
+	expected := []string{refDataTestCMToken}
+	annoVal, _ := json.Marshal(expected)
+	wd := makeWDForCell(string(annoVal))
+	inst := makeInstanceWithRefDataGate()
+	companionCM := makeCompanionConfigMap(refDataTestCMCompanionName)
+
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd, companionCM})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond)
+	assert.Equal(t, metav1.ConditionTrue, cond.Status,
+		"should be Ready=True when all companions are present and no terminal error")
+}
+
+// TestFederated_QuotaAndSourceNotFound_SourceNotFoundWins verifies that when
+// both the Quota gate and ReferencedData gate are present, and the cell WD
+// carries a terminal-error annotation, Instance.Ready picks SourceNotFound
+// (priority 5) over PendingQuota (priority 3).
+func TestFederated_QuotaAndSourceNotFound_SourceNotFoundWins(t *testing.T) {
+	msg := `ConfigMap "app-config" not found in namespace "default"`
+	wd := makeWDWithTerminalError(computev1alpha.ReferencedDataReasonSourceNotFound, msg)
+	inst := makeInstanceWithRefDataGate()
+
+	// Add the Quota gate alongside ReferencedData.
+	inst.Spec.Controller.SchedulingGates = append(inst.Spec.Controller.SchedulingGates,
+		computev1alpha.SchedulingGate{Name: instancecontrol.QuotaSchedulingGate.String()},
+	)
+
+	// Seed a QuotaGranted=False/QuotaExceeded condition on the instance.
+	inst.Status.Conditions = []metav1.Condition{
+		{
+			Type:               computev1alpha.InstanceQuotaGranted,
+			Status:             metav1.ConditionFalse,
+			Reason:             computev1alpha.InstanceQuotaGrantedReasonQuotaExceeded,
+			Message:            "Quota exceeded for project",
+			ObservedGeneration: 1,
+			LastTransitionTime: metav1.Now(),
+		},
+	}
+
+	r, projectClient, _ := newRefDataReconciler(t, []client.Object{inst, wd})
+	reconcileRefData(t, r)
+
+	var updated computev1alpha.Instance
+	require.NoError(t, projectClient.Get(context.Background(),
+		types.NamespacedName{Namespace: refDataTestNamespace, Name: refDataTestInstance}, &updated))
+
+	cond := apimeta.FindStatusCondition(updated.Status.Conditions, computev1alpha.ReferencedDataReady)
+	require.NotNil(t, cond)
+	assert.Equal(t, computev1alpha.ReferencedDataReasonSourceNotFound, cond.Reason,
+		"SourceNotFound should be set on ReferencedDataReady so reconcileGatedReadyCondition picks priority 5")
+
+	// Note: reconcileGatedReadyCondition (called by reconcileInstanceReadyCondition,
+	// not tested here) is where the priority competition between SourceNotFound (p5)
+	// and PendingQuota (p3) is resolved. This test verifies that reconcileReferencedDataCondition
+	// sets the right ReferencedDataReady sub-condition that feeds into that priority logic.
+}
+
 // containsAll returns true when s contains all substrings.
 func containsAll(s string, subs ...string) bool {
 	for _, sub := range subs {