Merge pull request #129 from datum-cloud/feat/configmap-secret-mounts-federated

Workloads can reference ConfigMaps and Secrets, delivered to every POP cell

Author: scotwells

api/v1alpha/annotations.go

@@ -4,4 +4,46 @@ const (
 	AnnotationNamespace = "compute.datumapis.com"
 
 	SSHKeysAnnotation = AnnotationNamespace + "/ssh-keys"
+
+	// ExpectedReferencedDataAnnotation is set on a WorkloadDeployment by the
+	// ReferencedDataController. Its value is a JSON-encoded array of companion
+	// object names (sorted deterministically) that the cell should expect.
+	// The cell does a pure set-membership check against labeled companions
+	// without recomputing names.
+	//
+	// Example value: ["configmap.app-config","secret.db-creds"]
+	ExpectedReferencedDataAnnotation = AnnotationNamespace + "/expected-referenced-data"
+
+	// RestartedAtAnnotation may be set on an InstanceTemplateSpec's annotations
+	// to trigger a rolling restart. The value is an RFC3339 timestamp. Because
+	// this annotation lives in the template metadata, it is included in the
+	// template hash and triggers the existing ordered in-place roll.
+	RestartedAtAnnotation = AnnotationNamespace + "/restartedAt"
+
+	// ReferencedDataGateStartAnnotation is stamped on an Instance by the cell
+	// InstanceReconciler the first time it observes the ReferencedData scheduling
+	// gate. Its value is an RFC3339 timestamp. Used to compute gate-wait duration
+	// for the compute_referenced_data_gate_wait_seconds histogram.
+	ReferencedDataGateStartAnnotation = AnnotationNamespace + "/referenced-data-gate-start"
+
+	// ReferencedDataErrorAnnotation is stamped on a WorkloadDeployment by the
+	// ReferencedDataController when a terminal source error occurs (SourceNotFound,
+	// SourceUnauthorized, or SourceTooLarge). Its value is a JSON object with
+	// "reason" and "message" fields carrying the authoritative resolver verdict.
+	//
+	// Example value:
+	//   {"reason":"SourceNotFound","message":"ConfigMap \"app-config\" not found in namespace \"default\""}
+	//
+	// This annotation bridges the federation boundary: Karmada propagates
+	// metadata.annotations hub→cell alongside WorkloadDeployment objects, but
+	// status.conditions do not propagate in that direction. The cell
+	// InstanceReconciler reads this annotation from the cell WD copy (returned by
+	// fetchOwnerWorkloadDeployment) and promotes it to the Instance's
+	// ReferencedDataReady condition so the terminal error is visible at the Instance
+	// level without requiring a cross-plane condition read.
+	//
+	// The annotation is removed when the error resolves (companion materialises /
+	// ReferencedDataReady flips True), so the absence of the annotation means
+	// either no error or the error has cleared.
+	ReferencedDataErrorAnnotation = AnnotationNamespace + "/referenced-data-error"
 )

api/v1alpha/instance_types.go

@@ -138,6 +138,16 @@ type SandboxContainer struct {
 	// so replicate the structure here too.
 	Env []corev1.EnvVar `json:"env,omitempty"`
 
+	// List of sources to populate environment variables in the container.
+	// The keys defined within a source must be a C_IDENTIFIER. All invalid
+	// keys will be reported as an event when the container is starting. When a
+	// key exists in multiple sources, the value associated with the last source
+	// will take precedence. Values defined by an Env with a duplicate key will
+	// take precedence.
+	//
+	// +kubebuilder:validation:Optional
+	EnvFrom []EnvFromSource `json:"envFrom,omitempty"`
+
 	// The resource requirements for the container, such as CPU, memory, and GPUs.
 	//
 	// +kubebuilder:validation:Optional
@@ -156,6 +166,54 @@ type SandboxContainer struct {
 	Ports []NamedPort `json:"ports,omitempty"`
 }
 
+// EnvFromSource represents a source for a set of ConfigMaps or Secrets to be
+// used as environment variables in a container.
+type EnvFromSource struct {
+	// An optional identifier to prepend to each key in the referenced
+	// ConfigMap or Secret. Must be a valid C_IDENTIFIER.
+	//
+	// +kubebuilder:validation:Optional
+	Prefix string `json:"prefix,omitempty"`
+
+	// The ConfigMap to select from.
+	//
+	// +kubebuilder:validation:Optional
+	ConfigMapRef *ConfigMapEnvSource `json:"configMapRef,omitempty"`
+
+	// The Secret to select from.
+	//
+	// +kubebuilder:validation:Optional
+	SecretRef *SecretEnvSource `json:"secretRef,omitempty"`
+}
+
+// ConfigMapEnvSource selects a ConfigMap to populate the environment variables
+// of a container.
+type ConfigMapEnvSource struct {
+	// Name of the ConfigMap in the same namespace as the Workload.
+	//
+	// +kubebuilder:validation:Required
+	Name string `json:"name"`
+
+	// Specify whether the ConfigMap must be defined.
+	//
+	// +kubebuilder:validation:Optional
+	Optional *bool `json:"optional,omitempty"`
+}
+
+// SecretEnvSource selects a Secret to populate the environment variables
+// of a container.
+type SecretEnvSource struct {
+	// Name of the Secret in the same namespace as the Workload.
+	//
+	// +kubebuilder:validation:Required
+	Name string `json:"name"`
+
+	// Specify whether the Secret must be defined.
+	//
+	// +kubebuilder:validation:Optional
+	Optional *bool `json:"optional,omitempty"`
+}
+
 type ContainerResourceRequirements struct {
 	// Limits describes the maximum amount of compute resources allowed.
 	//
@@ -414,6 +472,38 @@ const (
 
 	// InstanceQuotaGranted indicates whether quota has been allocated for the instance
 	InstanceQuotaGranted = "QuotaGranted"
+
+	// ReferencedDataReady indicates whether all ConfigMaps and Secrets referenced
+	// by the workload template have been resolved and delivered to the cell.
+	// This condition is set on both WorkloadDeployment (resolver view) and
+	// Instance (cell view).
+	ReferencedDataReady = "ReferencedDataReady"
+)
+
+const (
+	// ReferencedDataReasonResolving indicates the resolver is in the process of
+	// reading source ConfigMaps/Secrets from the project control plane.
+	ReferencedDataReasonResolving = "Resolving"
+
+	// ReferencedDataReasonAwaitingPropagation indicates the expected companions
+	// have not yet all arrived on the cell.
+	ReferencedDataReasonAwaitingPropagation = "AwaitingPropagation"
+
+	// ReferencedDataReasonSourceNotFound indicates one or more referenced
+	// ConfigMaps or Secrets could not be found in the project namespace.
+	ReferencedDataReasonSourceNotFound = "SourceNotFound"
+
+	// ReferencedDataReasonSourceUnauthorized indicates the management identity
+	// does not have permission to read one or more referenced objects.
+	ReferencedDataReasonSourceUnauthorized = "SourceUnauthorized"
+
+	// ReferencedDataReasonSourceTooLarge indicates one or more referenced objects
+	// exceed the allowed size limit.
+	ReferencedDataReasonSourceTooLarge = "SourceTooLarge"
+
+	// ReferencedDataReasonReady indicates all referenced data has been resolved
+	// and is present on the cell.
+	ReferencedDataReasonReady = "Ready"
 )
 
 const (

api/v1alpha/labels.go

@@ -23,4 +23,13 @@ const (
 	// PlacementNameLabel carries the placement name from the Workload that drove
 	// this Instance's deployment, sourced from WorkloadDeploymentSpec.PlacementName.
 	PlacementNameLabel = LabelNamespace + "/placement-name"
+
+	// ReferencedDataLabel is stamped on companion ConfigMaps and Secrets
+	// materialized by the ReferencedDataController, and on WorkloadDeployments
+	// that reference external ConfigMaps or Secrets. Used as a label selector
+	// by the Karmada PropagationPolicy to propagate companions to cells.
+	ReferencedDataLabel = LabelNamespace + "/referenced-data"
+
+	// ReferencedDataLabelValue is the value used for ReferencedDataLabel.
+	ReferencedDataLabelValue = "true"
 )

api/v1alpha/zz_generated.deepcopy.go

@@ -273,9 +273,12 @@ func main() {
 	}
 
 	if enableCellControllers {
+		wdOpts := controller.WorkloadDeploymentReconcilerOptions{
+			EnableReferencedDataGate: serverConfig.FeatureFlags.EnableReferencedDataGate,
+		}
 		if err = (&controller.WorkloadDeploymentReconciler{
 			NetworkingEnabled: features.FeatureGate.Enabled(features.NetworkingIntegration),
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, wdOpts); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "WorkloadDeployment")
 			os.Exit(1)
 		}
@@ -310,6 +313,29 @@ func main() {
 		}
 		runnables = append(runnables, extra...)
 	}
+	// ReferencedDataController is a management-plane controller (it reconciles
+	// WorkloadDeployments on project clusters and materialises companions). Gate
+	// it to the management controller set so it does not collide with the cell's
+	// WorkloadDeploymentReconciler.
+	if enableManagementControllers {
+		if err = (&controller.ReferencedDataController{}).SetupWithManager(mgr, controller.ReferencedDataControllerOptions{
+			// ProjectReader is nil for single-cluster mode; the controller falls back
+			// to a LocalReader. Set this to a *referenceddata.ProjectReader when the
+			// Milo multicluster mode is active and cross-project reads are required.
+			Reader: nil,
+			// FederationClient is set when the federation hub (Karmada) is configured.
+			// When non-nil, companions are materialised into the downstream
+			// ns-{project-uid} namespace on the hub so Karmada can propagate them
+			// to cells alongside the WorkloadDeployment. When nil, companions land
+			// in the project namespace (single-cluster / dev path).
+			FederationClient:    federationClient,
+			PerObjectLimitBytes: serverConfig.ReferencedData.PerObjectLimitBytes,
+			AggregateLimitBytes: serverConfig.ReferencedData.AggregateLimitBytes,
+		}); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "ReferencedData")
+			os.Exit(1)
+		}
+	}
 
 	if serverConfig.WebhookServer != nil {
 		if err = computev1alphawebhooks.SetupWorkloadWebhookWithManager(mgr); err != nil {

cmd/main.go

@@ -455,6 +455,54 @@ spec:
                               x-kubernetes-list-map-keys:
                               - name
                               x-kubernetes-list-type: map
+                            envFrom:
+                              description: |-
+                                List of sources to populate environment variables in the container.
+                                The keys defined within a source must be a C_IDENTIFIER. All invalid
+                                keys will be reported as an event when the container is starting. When a
+                                key exists in multiple sources, the value associated with the last source
+                                will take precedence. Values defined by an Env with a duplicate key will
+                                take precedence.
+                              items:
+                                description: |-
+                                  EnvFromSource represents a source for a set of ConfigMaps or Secrets to be
+                                  used as environment variables in a container.
+                                properties:
+                                  configMapRef:
+                                    description: The ConfigMap to select from.
+                                    properties:
+                                      name:
+                                        description: Name of the ConfigMap in the
+                                          same namespace as the Workload.
+                                        type: string
+                                      optional:
+                                        description: Specify whether the ConfigMap
+                                          must be defined.
+                                        type: boolean
+                                    required:
+                                    - name
+                                    type: object
+                                  prefix:
+                                    description: |-
+                                      An optional identifier to prepend to each key in the referenced
+                                      ConfigMap or Secret. Must be a valid C_IDENTIFIER.
+                                    type: string
+                                  secretRef:
+                                    description: The Secret to select from.
+                                    properties:
+                                      name:
+                                        description: Name of the Secret in the same
+                                          namespace as the Workload.
+                                        type: string
+                                      optional:
+                                        description: Specify whether the Secret must
+                                          be defined.
+                                        type: boolean
+                                    required:
+                                    - name
+                                    type: object
+                                type: object
+                              type: array
                             image:
                               description: The fully qualified container image name.
                               type: string