From b29bb9a680fc1e67b0bb05bf29f67a5da25963e4 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Mon, 29 Jun 2026 13:54:10 -0400 Subject: [PATCH 1/8] scaffold DGD placement score status wiring Signed-off-by: ashnamehrotra --- .../conversion_field_coverage_test.go | 2 + .../dynamographdeployment_conversion.go | 12 +++++ .../dynamographdeployment_conversion_test.go | 2 + .../v1alpha1/dynamographdeployment_types.go | 18 +++++++ .../api/v1alpha1/zz_generated.deepcopy.go | 5 ++ deploy/operator/api/v1beta1/common.go | 13 +++++ .../v1beta1/dynamographdeployment_types.go | 9 ++++ .../api/v1beta1/zz_generated.deepcopy.go | 5 ++ .../dynamographdeployment_controller.go | 14 ++++++ deploy/operator/internal/dynamo/grove.go | 27 ++++++++++ deploy/operator/internal/dynamo/grove_test.go | 49 +++++++++++++++++++ 11 files changed, 156 insertions(+) diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 927db4165d49..4298ada1c938 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -152,6 +152,8 @@ DynamoGraphDeploymentStatus.components.replicas DynamoGraphDeploymentStatus.components.updatedReplicas DynamoGraphDeploymentStatus.conditions DynamoGraphDeploymentStatus.observedGeneration +DynamoGraphDeploymentStatus.placementScore +DynamoGraphDeploymentStatus.placementScoreState DynamoGraphDeploymentStatus.restart.inProgress DynamoGraphDeploymentStatus.restart.observedID DynamoGraphDeploymentStatus.restart.phase diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 1428be9d1c8e..1459c1991899 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -632,6 +632,12 @@ func ConvertToSpecTopologyConstraint(src *v1beta1.SpecTopologyConstraint, dst *S func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, dst *v1beta1.DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = v1beta1.DGDState(src.State) + if src.PlacementScore != nil { + dst.PlacementScore = ptr.To(*src.PlacementScore) + } else { + dst.PlacementScore = nil + } + dst.PlacementScoreState = v1beta1.PlacementScoreState(src.PlacementScoreState) if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { @@ -669,6 +675,12 @@ func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, ds func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStatus, dst *DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = DGDState(src.State) + if src.PlacementScore != nil { + dst.PlacementScore = ptr.To(*src.PlacementScore) + } else { + dst.PlacementScore = nil + } + dst.PlacementScoreState = PlacementScoreState(src.PlacementScoreState) if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index cf786fa9ac20..ec46f97e9119 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -893,6 +893,8 @@ func TestDGD_RoundTrip_Status(t *testing.T) { Status: v1beta1.DynamoGraphDeploymentStatus{ ObservedGeneration: 7, State: v1beta1.DGDStateSuccessful, + PlacementScore: ptr.To(0.87), + PlacementScoreState: v1beta1.PlacementScoreStateUnsupported, Conditions: []metav1.Condition{ { Type: "Ready", diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 8f340cc5f4de..e961919d9be3 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -54,6 +54,17 @@ const ( DGDStateFailed DGDState = "failed" ) +// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;NotReported;Unknown +type PlacementScoreState string + +const ( + PlacementScoreStateReported PlacementScoreState = "Reported" + PlacementScoreStatePartial PlacementScoreState = "Partial" + PlacementScoreStateUnsupported PlacementScoreState = "Unsupported" + PlacementScoreStateNotReported PlacementScoreState = "NotReported" + PlacementScoreStateUnknown PlacementScoreState = "Unknown" +) + // DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment. type DynamoGraphDeploymentSpec struct { // Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). @@ -168,6 +179,13 @@ type DynamoGraphDeploymentStatus struct { // Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` + // PlacementScore is the DGD-level scheduler placement score aggregated from + // relevant scheduler placement units when reported. + // +optional + PlacementScore *float64 `json:"placementScore,omitempty"` + // PlacementScoreState indicates placement score reporting state. + // +optional + PlacementScoreState PlacementScoreState `json:"placementScoreState,omitempty"` } // ServiceCheckpointStatus contains checkpoint information for a single service. diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 2deae363eac7..67b5d579fbec 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1052,6 +1052,11 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(RollingUpdateStatus) (*in).DeepCopyInto(*out) } + if in.PlacementScore != nil { + in, out := &in.PlacementScore, &out.PlacementScore + *out = new(float64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index e0f2cf5d712b..cab13d4fcc9f 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -564,6 +564,19 @@ const ( DGDStateFailed DGDState = "failed" ) +// PlacementScoreState describes whether placement score is available and how +// complete the reported score is for a graph deployment. +// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;NotReported;Unknown +type PlacementScoreState string + +const ( + PlacementScoreStateReported PlacementScoreState = "Reported" + PlacementScoreStatePartial PlacementScoreState = "Partial" + PlacementScoreStateUnsupported PlacementScoreState = "Unsupported" + PlacementScoreStateNotReported PlacementScoreState = "NotReported" + PlacementScoreStateUnknown PlacementScoreState = "Unknown" +) + // RestartPhase enumerates phases of a graph-level restart. type RestartPhase string diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 17af6890d105..27db615a1c09 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -125,6 +125,15 @@ type DynamoGraphDeploymentStatus struct { // Currently only supported for single-node, non-Grove deployments (DCD/Deployment). // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` + + // placementScore is the DGD-level scheduler placement score aggregated from + // relevant scheduler placement units when reported. + // +optional + PlacementScore *float64 `json:"placementScore,omitempty"` + + // placementScoreState indicates placement score reporting state. + // +optional + PlacementScoreState PlacementScoreState `json:"placementScoreState,omitempty"` } // +kubebuilder:object:root=true diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index e1cdc47cf1f1..7ccefeb0d430 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -805,6 +805,11 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(RollingUpdateStatus) (*in).DeepCopyInto(*out) } + if in.PlacementScore != nil { + in, out := &in.PlacementScore, &out.PlacementScore + *out = new(float64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamoGraphDeploymentStatus. diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 5365b110afbc..e19a93a1781b 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -286,6 +286,20 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr dynamoDeployment.Status.Components = reconcileResult.ComponentStatus dynamoDeployment.Status.Restart = reconcileResult.RestartStatus + // Compute DGD-level placement score from the Grove PodCliqueSet when on + // the Grove pathway. Score is sourced from scheduler PodGang statuses and + // is purely informational — it does not affect readiness. + if r.isGrovePathway(dynamoDeployment) { + pcs, pcsErr := r.getExistingGrovePodCliqueSet(ctx, dynamoDeployment) + if pcsErr != nil { + logger.V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) + } else { + score, state := dynamo.AggregatePlacementScore(pcs) + dynamoDeployment.Status.PlacementScore = score + dynamoDeployment.Status.PlacementScoreState = state + } + } + if err != nil { logger.Error(err, "failed to reconcile the resources") reason = "failed_to_reconcile_the_resources" diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index 866d8e20e47f..ce650c0c4b4c 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -260,6 +260,33 @@ func CheckPCSGReady(ctx context.Context, client client.Client, resourceName, nam return true, "", serviceStatus } +// AggregatePlacementScore reads PodGangStatutes from the given PodCliqueSet and +// derives the DGD-level placement score. The score is sourced from the +// scheduler-reported PodGang placement score mirrored into +// PodCliqueSet.status.podGangStatuses. When multiple PodGangs report a score +// the minimum is used (conservative: one poorly placed unit lowers the DGD +// score). Absence is non-failing — it sets state to Unsupported or NotReported +// and does not touch the score pointer. +// +// NOTE: The current Grove alpha.8 PodGangStatus has no PlacementScore field. +// When Grove adds it, replace the TODO below with the actual field read and +// remove the forced Unsupported state. +func AggregatePlacementScore(pcs *grovev1alpha1.PodCliqueSet) (score *float64, state v1beta1.PlacementScoreState) { + if pcs == nil { + return nil, v1beta1.PlacementScoreStateUnsupported + } + + gangs := pcs.Status.PodGangStatutes + if len(gangs) == 0 { + return nil, v1beta1.PlacementScoreStateNotReported + } + + // TODO: read gang.PlacementScore once Grove PodGangStatus exposes the + // field. Until then, report Unsupported so absence is explicit and + // non-failing. + return nil, v1beta1.PlacementScoreStateUnsupported +} + // specToGroveTopologyConstraint converts a deployment-level SpecTopologyConstraint // to a Grove TopologyConstraint, extracting only the PackDomain. func specToGroveTopologyConstraint(tc *v1beta1.SpecTopologyConstraint) *grovev1alpha1.TopologyConstraint { diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index f1f5843326b6..13538b382af4 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -1188,3 +1188,52 @@ func Test_GetComponentReadinessAndServiceReplicaStatuses(t *testing.T) { }) } } + +func TestAggregatePlacementScore_CurrentBehavior(t *testing.T) { + tests := []struct { + name string + pcs *grovev1alpha1.PodCliqueSet + wantScore *float64 + wantState v1beta1.PlacementScoreState + }{ + { + name: "nil pod clique set", + pcs: nil, + wantScore: nil, + wantState: v1beta1.PlacementScoreStateUnsupported, + }, + { + name: "no pod gangs reported", + pcs: &grovev1alpha1.PodCliqueSet{ + Status: grovev1alpha1.PodCliqueSetStatus{}, + }, + wantScore: nil, + wantState: v1beta1.PlacementScoreStateNotReported, + }, + { + name: "pod gangs present but no score field in current grove api", + pcs: &grovev1alpha1.PodCliqueSet{ + Status: grovev1alpha1.PodCliqueSetStatus{ + PodGangStatutes: []grovev1alpha1.PodGangStatus{{ + Name: "pg-0", + Phase: grovev1alpha1.PodGangRunning, + }}, + }, + }, + wantScore: nil, + wantState: v1beta1.PlacementScoreStateUnsupported, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + score, state := AggregatePlacementScore(tt.pcs) + if score != nil || tt.wantScore != nil { + t.Fatalf("score = %v, want %v", score, tt.wantScore) + } + if state != tt.wantState { + t.Fatalf("state = %q, want %q", state, tt.wantState) + } + }) + } +} From 0c56c9e5e59ddf3097786d611f60ea74712de911 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 30 Jun 2026 09:39:05 -0400 Subject: [PATCH 2/8] fix(operator): resolve pre-merge operator lint failures Signed-off-by: ashnamehrotra --- .../dynamographdeployment_conversion_test.go | 6 ++-- .../dynamographdeployment_controller.go | 28 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index ec46f97e9119..d13a3e883bbc 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -891,9 +891,9 @@ func TestDGD_RoundTrip_Status(t *testing.T) { src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "status", Namespace: "ns"}, Status: v1beta1.DynamoGraphDeploymentStatus{ - ObservedGeneration: 7, - State: v1beta1.DGDStateSuccessful, - PlacementScore: ptr.To(0.87), + ObservedGeneration: 7, + State: v1beta1.DGDStateSuccessful, + PlacementScore: ptr.To(0.87), PlacementScoreState: v1beta1.PlacementScoreStateUnsupported, Conditions: []metav1.Condition{ { diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index e19a93a1781b..28cd22341102 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -25,6 +25,7 @@ import ( groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" + "github.com/go-logr/logr" "github.com/imdario/mergo" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" @@ -289,16 +290,7 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr // Compute DGD-level placement score from the Grove PodCliqueSet when on // the Grove pathway. Score is sourced from scheduler PodGang statuses and // is purely informational — it does not affect readiness. - if r.isGrovePathway(dynamoDeployment) { - pcs, pcsErr := r.getExistingGrovePodCliqueSet(ctx, dynamoDeployment) - if pcsErr != nil { - logger.V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) - } else { - score, state := dynamo.AggregatePlacementScore(pcs) - dynamoDeployment.Status.PlacementScore = score - dynamoDeployment.Status.PlacementScoreState = state - } - } + r.updatePlacementScoreStatus(ctx, logger, dynamoDeployment) if err != nil { logger.Error(err, "failed to reconcile the resources") @@ -704,6 +696,22 @@ func (r *DynamoGraphDeploymentReconciler) getExistingGrovePodCliqueSet(ctx conte return pcs, nil } +func (r *DynamoGraphDeploymentReconciler) updatePlacementScoreStatus(ctx context.Context, logger logr.Logger, dgd *nvidiacomv1beta1.DynamoGraphDeployment) { + if !r.isGrovePathway(dgd) { + return + } + + pcs, pcsErr := r.getExistingGrovePodCliqueSet(ctx, dgd) + if pcsErr != nil { + logger.V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) + return + } + + score, state := dynamo.AggregatePlacementScore(pcs) + dgd.Status.PlacementScore = score + dgd.Status.PlacementScoreState = state +} + func restartAnnotationsFromPodCliqueSet(pcs *grovev1alpha1.PodCliqueSet) map[string]string { restartAnnotations := make(map[string]string) if pcs == nil { From de57d0f3762aaa402944137110b24f9d5141e2b6 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 30 Jun 2026 12:33:11 -0400 Subject: [PATCH 3/8] fix(operator): regenerate CRDs and api docs for placement-score fields Signed-off-by: ashnamehrotra --- .../nvidia.com_dynamographdeployments.yaml | 28 +++++++++++ .../nvidia.com_dynamographdeployments.yaml | 28 +++++++++++ docs/kubernetes/api-reference.md | 47 +++++++++++++++++++ 3 files changed, 103 insertions(+) diff --git a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml index 0d4371ed5518..e4c795a4527d 100644 --- a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml +++ b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml @@ -11947,6 +11947,20 @@ spec: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 type: integer + placementScore: + description: |- + PlacementScore is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units when reported. + type: number + placementScoreState: + description: PlacementScoreState indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - NotReported + - Unknown + type: string restart: description: Restart contains the status of the restart of the graph deployment. properties: @@ -21313,6 +21327,20 @@ spec: description: observedGeneration is the most recent generation observed by the controller. format: int64 type: integer + placementScore: + description: |- + placementScore is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units when reported. + type: number + placementScoreState: + description: placementScoreState indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - NotReported + - Unknown + type: string restart: description: restart contains the status of a graph-level restart. properties: diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index 0d4371ed5518..e4c795a4527d 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11947,6 +11947,20 @@ spec: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 type: integer + placementScore: + description: |- + PlacementScore is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units when reported. + type: number + placementScoreState: + description: PlacementScoreState indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - NotReported + - Unknown + type: string restart: description: Restart contains the status of the restart of the graph deployment. properties: @@ -21313,6 +21327,20 @@ spec: description: observedGeneration is the most recent generation observed by the controller. format: int64 type: integer + placementScore: + description: |- + placementScore is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units when reported. + type: number + placementScoreState: + description: placementScoreState indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - NotReported + - Unknown + type: string restart: description: restart contains the status of a graph-level restart. properties: diff --git a/docs/kubernetes/api-reference.md b/docs/kubernetes/api-reference.md index 192138e8222d..ea8ead5d1008 100644 --- a/docs/kubernetes/api-reference.md +++ b/docs/kubernetes/api-reference.md @@ -737,6 +737,8 @@ _Appears in:_ | `restart` _[RestartStatus](#restartstatus)_ | Restart contains the status of the restart of the graph deployment. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| +| `placementScore` _float_ | PlacementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units when reported. | | Optional: \{\}
| +| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | PlacementScoreState indicates placement score reporting state. | | Enum: [Reported Partial Unsupported NotReported Unknown]
Optional: \{\}
| #### DynamoModel @@ -1126,6 +1128,27 @@ _Appears in:_ | `volumeAccessMode` _[PersistentVolumeAccessMode](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#persistentvolumeaccessmode-v1-core)_ | VolumeAccessMode is the volume access mode of the PVC. Required when create is true. | | | +#### PlacementScoreState + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Reported Partial Unsupported NotReported Unknown] + +_Appears in:_ +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + +| Field | Description | +| --- | --- | +| `Reported` | | +| `Partial` | | +| `Unsupported` | | +| `NotReported` | | +| `Unknown` | | + + #### PodReference @@ -2371,6 +2394,8 @@ _Appears in:_ | `restart` _[RestartStatus](#restartstatus)_ | restart contains the status of a graph-level restart. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| +| `placementScore` _float_ | placementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units when reported. | | Optional: \{\}
| +| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | placementScoreState indicates placement score reporting state. | | Enum: [Reported Partial Unsupported NotReported Unknown]
Optional: \{\}
| #### EPPConfig @@ -2737,6 +2762,28 @@ _Appears in:_ | `config` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Config is the full deployment configuration for this Pareto point. | | Type: object
| +#### PlacementScoreState + +_Underlying type:_ _string_ + +PlacementScoreState describes whether placement score is available and how +complete the reported score is for a graph deployment. + +_Validation:_ +- Enum: [Reported Partial Unsupported NotReported Unknown] + +_Appears in:_ +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + +| Field | Description | +| --- | --- | +| `Reported` | | +| `Partial` | | +| `Unsupported` | | +| `NotReported` | | +| `Unknown` | | + + #### ProfilingPhase _Underlying type:_ _string_ From d86e8fdc87da318069cd8071687c200e3e742a49 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 8 Jul 2026 10:54:18 -0400 Subject: [PATCH 4/8] feat(operator): align DGD placement score with updated DEP - Drop the NotReported PlacementScoreState (removed by DEP; not operationally distinct from Unknown). Empty PodGang statuses now report Unknown. - Enforce the [0.0, 1.0] score contract at the CRD boundary via Minimum=0 / Maximum=1 kubebuilder markers on the placementScore field in both v1alpha1 and v1beta1. - Document the score contract on the API field: normalized [0,1], higher-is-better, minimum aggregation as a worst-placement signal, comparability only within the same scheduler scoring contract. - Fix the reporting lifecycle so every reconcile leaves status consistent: * non-Grove pathway sets Unsupported and clears any stale score, * PCS read failure sets Unknown and clears any stale score, * successful aggregation writes through unchanged. - Regenerate CRDs, api-reference, and the operator Helm chart CRD copy. Signed-off-by: ashnamehrotra --- .../nvidia.com_dynamographdeployments.yaml | 26 +++++++++++++----- .../v1alpha1/dynamographdeployment_types.go | 17 +++++++++--- deploy/operator/api/v1beta1/common.go | 12 +++++++-- .../v1beta1/dynamographdeployment_types.go | 11 ++++++-- .../nvidia.com_dynamographdeployments.yaml | 26 +++++++++++++----- .../dynamographdeployment_controller.go | 13 +++++++++ deploy/operator/internal/dynamo/grove.go | 27 ++++++++++++------- deploy/operator/internal/dynamo/grove_test.go | 2 +- docs/kubernetes/api-reference.md | 26 +++++++++++------- 9 files changed, 120 insertions(+), 40 deletions(-) diff --git a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml index e4c795a4527d..d92617d51af6 100644 --- a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml +++ b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml @@ -11950,15 +11950,22 @@ spec: placementScore: description: |- PlacementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units when reported. + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation uses + the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 type: number placementScoreState: - description: PlacementScoreState indicates placement score reporting state. + description: |- + PlacementScoreState indicates placement score reporting state. See + PlacementScoreState for the semantics of each value. enum: - Reported - Partial - Unsupported - - NotReported - Unknown type: string restart: @@ -21330,15 +21337,22 @@ spec: placementScore: description: |- placementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units when reported. + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation uses + the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 type: number placementScoreState: - description: placementScoreState indicates placement score reporting state. + description: |- + placementScoreState indicates placement score reporting state. See + PlacementScoreState for the semantics of each value. enum: - Reported - Partial - Unsupported - - NotReported - Unknown type: string restart: diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index e961919d9be3..91f6fb9bae6a 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -54,14 +54,16 @@ const ( DGDStateFailed DGDState = "failed" ) -// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;NotReported;Unknown +// PlacementScoreState describes whether placement score is available and how +// complete the reported score is for a graph deployment. See the v1beta1 +// PlacementScoreState for the authoritative semantics of each value. +// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;Unknown type PlacementScoreState string const ( PlacementScoreStateReported PlacementScoreState = "Reported" PlacementScoreStatePartial PlacementScoreState = "Partial" PlacementScoreStateUnsupported PlacementScoreState = "Unsupported" - PlacementScoreStateNotReported PlacementScoreState = "NotReported" PlacementScoreStateUnknown PlacementScoreState = "Unknown" ) @@ -180,10 +182,17 @@ type DynamoGraphDeploymentStatus struct { // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` // PlacementScore is the DGD-level scheduler placement score aggregated from - // relevant scheduler placement units when reported. + // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + // is better and 1.0 represents the best possible placement. Aggregation uses + // the minimum across placement units so the value is a worst-placement + // signal for the graph. Scores are only comparable across DGDs that share + // the same scheduler scoring contract and version. // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 PlacementScore *float64 `json:"placementScore,omitempty"` - // PlacementScoreState indicates placement score reporting state. + // PlacementScoreState indicates placement score reporting state. See + // PlacementScoreState for the semantics of each value. // +optional PlacementScoreState PlacementScoreState `json:"placementScoreState,omitempty"` } diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index cab13d4fcc9f..48d9c80337df 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -566,14 +566,22 @@ const ( // PlacementScoreState describes whether placement score is available and how // complete the reported score is for a graph deployment. -// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;NotReported;Unknown +// +// Every backend must set this field after the first reconciliation: +// - Reported: a score is available for every scored placement unit. +// - Partial: a score is available for some but not all placement units. +// - Unsupported: the backend does not surface a placement score at all. +// - Unknown: the backend supports scores but the current value is +// indeterminate (e.g. read failure, not yet populated by the +// scheduler). When set, PlacementScore must be cleared. +// +// +kubebuilder:validation:Enum=Reported;Partial;Unsupported;Unknown type PlacementScoreState string const ( PlacementScoreStateReported PlacementScoreState = "Reported" PlacementScoreStatePartial PlacementScoreState = "Partial" PlacementScoreStateUnsupported PlacementScoreState = "Unsupported" - PlacementScoreStateNotReported PlacementScoreState = "NotReported" PlacementScoreStateUnknown PlacementScoreState = "Unknown" ) diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 27db615a1c09..2a58d265e90c 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -127,11 +127,18 @@ type DynamoGraphDeploymentStatus struct { RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` // placementScore is the DGD-level scheduler placement score aggregated from - // relevant scheduler placement units when reported. + // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + // is better and 1.0 represents the best possible placement. Aggregation uses + // the minimum across placement units so the value is a worst-placement + // signal for the graph. Scores are only comparable across DGDs that share + // the same scheduler scoring contract and version. // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 PlacementScore *float64 `json:"placementScore,omitempty"` - // placementScoreState indicates placement score reporting state. + // placementScoreState indicates placement score reporting state. See + // PlacementScoreState for the semantics of each value. // +optional PlacementScoreState PlacementScoreState `json:"placementScoreState,omitempty"` } diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index e4c795a4527d..d92617d51af6 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11950,15 +11950,22 @@ spec: placementScore: description: |- PlacementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units when reported. + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation uses + the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 type: number placementScoreState: - description: PlacementScoreState indicates placement score reporting state. + description: |- + PlacementScoreState indicates placement score reporting state. See + PlacementScoreState for the semantics of each value. enum: - Reported - Partial - Unsupported - - NotReported - Unknown type: string restart: @@ -21330,15 +21337,22 @@ spec: placementScore: description: |- placementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units when reported. + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation uses + the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 type: number placementScoreState: - description: placementScoreState indicates placement score reporting state. + description: |- + placementScoreState indicates placement score reporting state. See + PlacementScoreState for the semantics of each value. enum: - Reported - Partial - Unsupported - - NotReported - Unknown type: string restart: diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 28cd22341102..4d8014e6409c 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -696,14 +696,27 @@ func (r *DynamoGraphDeploymentReconciler) getExistingGrovePodCliqueSet(ctx conte return pcs, nil } +// updatePlacementScoreStatus writes the placement score fields on every +// reconcile so status is never left stale. Per the placement-score DEP, +// every backend must set placementScoreState after the first reconciliation: +// - non-Grove pathways report Unsupported (no placement score source), +// - Grove read failures report Unknown (indeterminate; may recover), +// - otherwise the aggregator's classification is written through. +// +// In every non-Reported/Partial branch PlacementScore is cleared so a value +// from a previous reconcile cannot linger. func (r *DynamoGraphDeploymentReconciler) updatePlacementScoreStatus(ctx context.Context, logger logr.Logger, dgd *nvidiacomv1beta1.DynamoGraphDeployment) { if !r.isGrovePathway(dgd) { + dgd.Status.PlacementScore = nil + dgd.Status.PlacementScoreState = nvidiacomv1beta1.PlacementScoreStateUnsupported return } pcs, pcsErr := r.getExistingGrovePodCliqueSet(ctx, dgd) if pcsErr != nil { logger.V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) + dgd.Status.PlacementScore = nil + dgd.Status.PlacementScoreState = nvidiacomv1beta1.PlacementScoreStateUnknown return } diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index ce650c0c4b4c..ba0b18688de3 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -260,17 +260,24 @@ func CheckPCSGReady(ctx context.Context, client client.Client, resourceName, nam return true, "", serviceStatus } -// AggregatePlacementScore reads PodGangStatutes from the given PodCliqueSet and -// derives the DGD-level placement score. The score is sourced from the -// scheduler-reported PodGang placement score mirrored into -// PodCliqueSet.status.podGangStatuses. When multiple PodGangs report a score -// the minimum is used (conservative: one poorly placed unit lowers the DGD -// score). Absence is non-failing — it sets state to Unsupported or NotReported -// and does not touch the score pointer. +// AggregatePlacementScore reads PodGangStatutes from the given PodCliqueSet +// and derives the DGD-level placement score. This implements the "Grove +// mirror" placement-score source alternative; the source of truth is still +// under discussion (see the placement-score DEP Open Questions). +// +// When multiple PodGangs report a score the minimum is used: the DGD-level +// value is a worst-placement signal, matching the DEP's rationale that one +// poorly placed serving unit should not be hidden by a weighted average. +// +// Absence is non-failing: +// - nil PodCliqueSet: Unsupported (no Grove object to read from). +// - PodGangStatutes empty: Unknown (Grove hasn't populated the status yet, +// which is indeterminate rather than a permanent absence). // // NOTE: The current Grove alpha.8 PodGangStatus has no PlacementScore field. -// When Grove adds it, replace the TODO below with the actual field read and -// remove the forced Unsupported state. +// Until Grove exposes it, this function reports Unsupported when gangs are +// present. Replace the TODO below with the actual field read (and any +// partial-report handling) once available. func AggregatePlacementScore(pcs *grovev1alpha1.PodCliqueSet) (score *float64, state v1beta1.PlacementScoreState) { if pcs == nil { return nil, v1beta1.PlacementScoreStateUnsupported @@ -278,7 +285,7 @@ func AggregatePlacementScore(pcs *grovev1alpha1.PodCliqueSet) (score *float64, s gangs := pcs.Status.PodGangStatutes if len(gangs) == 0 { - return nil, v1beta1.PlacementScoreStateNotReported + return nil, v1beta1.PlacementScoreStateUnknown } // TODO: read gang.PlacementScore once Grove PodGangStatus exposes the diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index 13538b382af4..3d182721bb6c 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -1208,7 +1208,7 @@ func TestAggregatePlacementScore_CurrentBehavior(t *testing.T) { Status: grovev1alpha1.PodCliqueSetStatus{}, }, wantScore: nil, - wantState: v1beta1.PlacementScoreStateNotReported, + wantState: v1beta1.PlacementScoreStateUnknown, }, { name: "pod gangs present but no score field in current grove api", diff --git a/docs/kubernetes/api-reference.md b/docs/kubernetes/api-reference.md index ea8ead5d1008..930fc36592ca 100644 --- a/docs/kubernetes/api-reference.md +++ b/docs/kubernetes/api-reference.md @@ -737,8 +737,8 @@ _Appears in:_ | `restart` _[RestartStatus](#restartstatus)_ | Restart contains the status of the restart of the graph deployment. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| -| `placementScore` _float_ | PlacementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units when reported. | | Optional: \{\}
| -| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | PlacementScoreState indicates placement score reporting state. | | Enum: [Reported Partial Unsupported NotReported Unknown]
Optional: \{\}
| +| `placementScore` _float_ | PlacementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation uses
the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | PlacementScoreState indicates placement score reporting state. See
PlacementScoreState for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| #### DynamoModel @@ -1132,10 +1132,12 @@ _Appears in:_ _Underlying type:_ _string_ - +PlacementScoreState describes whether placement score is available and how +complete the reported score is for a graph deployment. See the v1beta1 +PlacementScoreState for the authoritative semantics of each value. _Validation:_ -- Enum: [Reported Partial Unsupported NotReported Unknown] +- Enum: [Reported Partial Unsupported Unknown] _Appears in:_ - [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) @@ -1145,7 +1147,6 @@ _Appears in:_ | `Reported` | | | `Partial` | | | `Unsupported` | | -| `NotReported` | | | `Unknown` | | @@ -2394,8 +2395,8 @@ _Appears in:_ | `restart` _[RestartStatus](#restartstatus)_ | restart contains the status of a graph-level restart. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| -| `placementScore` _float_ | placementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units when reported. | | Optional: \{\}
| -| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | placementScoreState indicates placement score reporting state. | | Enum: [Reported Partial Unsupported NotReported Unknown]
Optional: \{\}
| +| `placementScore` _float_ | placementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation uses
the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | placementScoreState indicates placement score reporting state. See
PlacementScoreState for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| #### EPPConfig @@ -2769,8 +2770,16 @@ _Underlying type:_ _string_ PlacementScoreState describes whether placement score is available and how complete the reported score is for a graph deployment. +Every backend must set this field after the first reconciliation: + - Reported: a score is available for every scored placement unit. + - Partial: a score is available for some but not all placement units. + - Unsupported: the backend does not surface a placement score at all. + - Unknown: the backend supports scores but the current value is + indeterminate (e.g. read failure, not yet populated by the + scheduler). When set, PlacementScore must be cleared. + _Validation:_ -- Enum: [Reported Partial Unsupported NotReported Unknown] +- Enum: [Reported Partial Unsupported Unknown] _Appears in:_ - [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) @@ -2780,7 +2789,6 @@ _Appears in:_ | `Reported` | | | `Partial` | | | `Unsupported` | | -| `NotReported` | | | `Unknown` | | From a93fe915e3b3f350cd81c0240a7da0f9100c98a2 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Thu, 9 Jul 2026 16:00:35 -0400 Subject: [PATCH 5/8] refactor(operator): group DGD placement fields under status.placement Address review feedback from @sttts on PR #11057: - Introduce PlacementStatus struct (score + state) on both v1alpha1 and v1beta1 DGDStatus, replacing the flat placementScore / placementScoreState fields. This future-proofs the schema against new placement signals (scheduler contract version, last-report timestamp, per-unit reports) that would otherwise be a schema break to add later. - Update the operator helper to read logr.Logger from ctx via log.FromContext(ctx) instead of taking it as a parameter, and drop the now-unused go-logr/logr import. - Rename helper to updatePlacementStatus and write through a fresh PlacementStatus value so stale fields are never carried across reconciles. - Regenerate CRDs, api-reference, helm CRD copy, and deepcopy. Signed-off-by: ashnamehrotra --- .../nvidia.com_dynamographdeployments.yaml | 86 ++++++++++--------- .../conversion_field_coverage_test.go | 4 +- .../dynamographdeployment_conversion.go | 24 ++++-- .../dynamographdeployment_conversion_test.go | 10 ++- .../v1alpha1/dynamographdeployment_types.go | 31 ++++--- .../api/v1alpha1/zz_generated.deepcopy.go | 28 +++++- deploy/operator/api/v1beta1/common.go | 23 ++++- .../v1beta1/dynamographdeployment_types.go | 17 +--- .../api/v1beta1/zz_generated.deepcopy.go | 28 +++++- .../nvidia.com_dynamographdeployments.yaml | 86 ++++++++++--------- .../dynamographdeployment_controller.go | 31 ++++--- docs/kubernetes/api-reference.md | 52 +++++++++-- 12 files changed, 268 insertions(+), 152 deletions(-) diff --git a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml index d92617d51af6..c8158442a69a 100644 --- a/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml +++ b/deploy/helm/charts/platform/components/operator/crds/nvidia.com_dynamographdeployments.yaml @@ -11947,27 +11947,27 @@ spec: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 type: integer - placementScore: + placement: description: |- - PlacementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation uses - the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - placementScoreState: - description: |- - PlacementScoreState indicates placement score reporting state. See - PlacementScoreState for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string + Placement groups DGD-level scheduler placement signals (score, reporting + state, and any future placement fields). + properties: + score: + description: |- + Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] + where higher is better and 1.0 is the best possible placement. + maximum: 1 + minimum: 0 + type: number + state: + description: State indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object restart: description: Restart contains the status of the restart of the graph deployment. properties: @@ -21334,27 +21334,33 @@ spec: description: observedGeneration is the most recent generation observed by the controller. format: int64 type: integer - placementScore: - description: |- - placementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation uses - the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - placementScoreState: + placement: description: |- - placementScoreState indicates placement score reporting state. See - PlacementScoreState for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string + placement groups DGD-level scheduler placement signals (score, reporting + state, and any future placement fields). + properties: + score: + description: |- + score is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation + uses the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 + type: number + state: + description: |- + state indicates placement score reporting state. See PlacementScoreState + for the semantics of each value. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object restart: description: restart contains the status of a graph-level restart. properties: diff --git a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go index 4298ada1c938..266e2a7f2cba 100644 --- a/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go +++ b/deploy/operator/api/v1alpha1/conversion_field_coverage_test.go @@ -152,8 +152,8 @@ DynamoGraphDeploymentStatus.components.replicas DynamoGraphDeploymentStatus.components.updatedReplicas DynamoGraphDeploymentStatus.conditions DynamoGraphDeploymentStatus.observedGeneration -DynamoGraphDeploymentStatus.placementScore -DynamoGraphDeploymentStatus.placementScoreState +DynamoGraphDeploymentStatus.placement.score +DynamoGraphDeploymentStatus.placement.state DynamoGraphDeploymentStatus.restart.inProgress DynamoGraphDeploymentStatus.restart.observedID DynamoGraphDeploymentStatus.restart.phase diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go index 1459c1991899..6dac2cbbda4b 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion.go @@ -632,12 +632,16 @@ func ConvertToSpecTopologyConstraint(src *v1beta1.SpecTopologyConstraint, dst *S func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, dst *v1beta1.DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = v1beta1.DGDState(src.State) - if src.PlacementScore != nil { - dst.PlacementScore = ptr.To(*src.PlacementScore) + if src.Placement != nil { + dst.Placement = &v1beta1.PlacementStatus{ + State: v1beta1.PlacementScoreState(src.Placement.State), + } + if src.Placement.Score != nil { + dst.Placement.Score = ptr.To(*src.Placement.Score) + } } else { - dst.PlacementScore = nil + dst.Placement = nil } - dst.PlacementScoreState = v1beta1.PlacementScoreState(src.PlacementScoreState) if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { @@ -675,12 +679,16 @@ func ConvertFromDynamoGraphDeploymentStatus(src *DynamoGraphDeploymentStatus, ds func ConvertToDynamoGraphDeploymentStatus(src *v1beta1.DynamoGraphDeploymentStatus, dst *DynamoGraphDeploymentStatus) { dst.ObservedGeneration = src.ObservedGeneration dst.State = DGDState(src.State) - if src.PlacementScore != nil { - dst.PlacementScore = ptr.To(*src.PlacementScore) + if src.Placement != nil { + dst.Placement = &PlacementStatus{ + State: PlacementScoreState(src.Placement.State), + } + if src.Placement.Score != nil { + dst.Placement.Score = ptr.To(*src.Placement.Score) + } } else { - dst.PlacementScore = nil + dst.Placement = nil } - dst.PlacementScoreState = PlacementScoreState(src.PlacementScoreState) if len(src.Conditions) > 0 { dst.Conditions = make([]metav1.Condition, 0, len(src.Conditions)) for _, c := range src.Conditions { diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go index d13a3e883bbc..5ae8cec0c5eb 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_conversion_test.go @@ -891,10 +891,12 @@ func TestDGD_RoundTrip_Status(t *testing.T) { src := &v1beta1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{Name: "status", Namespace: "ns"}, Status: v1beta1.DynamoGraphDeploymentStatus{ - ObservedGeneration: 7, - State: v1beta1.DGDStateSuccessful, - PlacementScore: ptr.To(0.87), - PlacementScoreState: v1beta1.PlacementScoreStateUnsupported, + ObservedGeneration: 7, + State: v1beta1.DGDStateSuccessful, + Placement: &v1beta1.PlacementStatus{ + Score: ptr.To(0.87), + State: v1beta1.PlacementScoreStateReported, + }, Conditions: []metav1.Condition{ { Type: "Ready", diff --git a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go index 91f6fb9bae6a..cc90d6192bc8 100644 --- a/deploy/operator/api/v1alpha1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1alpha1/dynamographdeployment_types.go @@ -67,6 +67,21 @@ const ( PlacementScoreStateUnknown PlacementScoreState = "Unknown" ) +// PlacementStatus groups DGD-level scheduler placement fields under a single +// status object so future placement signals can be added without a schema +// break. See the v1beta1 PlacementStatus for the authoritative field docs. +type PlacementStatus struct { + // Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] + // where higher is better and 1.0 is the best possible placement. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + Score *float64 `json:"score,omitempty"` + // State indicates placement score reporting state. + // +optional + State PlacementScoreState `json:"state,omitempty"` +} + // DynamoGraphDeploymentSpec defines the desired state of DynamoGraphDeployment. type DynamoGraphDeploymentSpec struct { // Annotations to propagate to all child resources (PCS, DCD, Deployments, and pod templates). @@ -181,20 +196,10 @@ type DynamoGraphDeploymentStatus struct { // Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` - // PlacementScore is the DGD-level scheduler placement score aggregated from - // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - // is better and 1.0 represents the best possible placement. Aggregation uses - // the minimum across placement units so the value is a worst-placement - // signal for the graph. Scores are only comparable across DGDs that share - // the same scheduler scoring contract and version. - // +optional - // +kubebuilder:validation:Minimum=0 - // +kubebuilder:validation:Maximum=1 - PlacementScore *float64 `json:"placementScore,omitempty"` - // PlacementScoreState indicates placement score reporting state. See - // PlacementScoreState for the semantics of each value. + // Placement groups DGD-level scheduler placement signals (score, reporting + // state, and any future placement fields). // +optional - PlacementScoreState PlacementScoreState `json:"placementScoreState,omitempty"` + Placement *PlacementStatus `json:"placement,omitempty"` } // ServiceCheckpointStatus contains checkpoint information for a single service. diff --git a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 67b5d579fbec..c81f61edbcfa 100644 --- a/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1052,10 +1052,10 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(RollingUpdateStatus) (*in).DeepCopyInto(*out) } - if in.PlacementScore != nil { - in, out := &in.PlacementScore, &out.PlacementScore - *out = new(float64) - **out = **in + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(PlacementStatus) + (*in).DeepCopyInto(*out) } } @@ -1534,6 +1534,26 @@ func (in *PVC) DeepCopy() *PVC { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { + *out = *in + if in.Score != nil { + in, out := &in.Score, &out.Score + *out = new(float64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. +func (in *PlacementStatus) DeepCopy() *PlacementStatus { + if in == nil { + return nil + } + out := new(PlacementStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodReference) DeepCopyInto(out *PodReference) { *out = *in diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index 48d9c80337df..e4ab79c6d9a8 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -573,7 +573,7 @@ const ( // - Unsupported: the backend does not surface a placement score at all. // - Unknown: the backend supports scores but the current value is // indeterminate (e.g. read failure, not yet populated by the -// scheduler). When set, PlacementScore must be cleared. +// scheduler). When set, PlacementStatus.Score must be cleared. // // +kubebuilder:validation:Enum=Reported;Partial;Unsupported;Unknown type PlacementScoreState string @@ -585,6 +585,27 @@ const ( PlacementScoreStateUnknown PlacementScoreState = "Unknown" ) +// PlacementStatus groups DGD-level scheduler placement fields under a single +// status object so future placement signals (e.g. scheduler contract version, +// last-report timestamp, per-unit reports) can be added without a schema break. +type PlacementStatus struct { + // score is the DGD-level scheduler placement score aggregated from + // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + // is better and 1.0 represents the best possible placement. Aggregation + // uses the minimum across placement units so the value is a worst-placement + // signal for the graph. Scores are only comparable across DGDs that share + // the same scheduler scoring contract and version. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + Score *float64 `json:"score,omitempty"` + + // state indicates placement score reporting state. See PlacementScoreState + // for the semantics of each value. + // +optional + State PlacementScoreState `json:"state,omitempty"` +} + // RestartPhase enumerates phases of a graph-level restart. type RestartPhase string diff --git a/deploy/operator/api/v1beta1/dynamographdeployment_types.go b/deploy/operator/api/v1beta1/dynamographdeployment_types.go index 2a58d265e90c..74022f952522 100644 --- a/deploy/operator/api/v1beta1/dynamographdeployment_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeployment_types.go @@ -126,21 +126,10 @@ type DynamoGraphDeploymentStatus struct { // +optional RollingUpdate *RollingUpdateStatus `json:"rollingUpdate,omitempty"` - // placementScore is the DGD-level scheduler placement score aggregated from - // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - // is better and 1.0 represents the best possible placement. Aggregation uses - // the minimum across placement units so the value is a worst-placement - // signal for the graph. Scores are only comparable across DGDs that share - // the same scheduler scoring contract and version. + // placement groups DGD-level scheduler placement signals (score, reporting + // state, and any future placement fields). // +optional - // +kubebuilder:validation:Minimum=0 - // +kubebuilder:validation:Maximum=1 - PlacementScore *float64 `json:"placementScore,omitempty"` - - // placementScoreState indicates placement score reporting state. See - // PlacementScoreState for the semantics of each value. - // +optional - PlacementScoreState PlacementScoreState `json:"placementScoreState,omitempty"` + Placement *PlacementStatus `json:"placement,omitempty"` } // +kubebuilder:object:root=true diff --git a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go index 7ccefeb0d430..e00d45ddf2bd 100644 --- a/deploy/operator/api/v1beta1/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta1/zz_generated.deepcopy.go @@ -805,10 +805,10 @@ func (in *DynamoGraphDeploymentStatus) DeepCopyInto(out *DynamoGraphDeploymentSt *out = new(RollingUpdateStatus) (*in).DeepCopyInto(*out) } - if in.PlacementScore != nil { - in, out := &in.PlacementScore, &out.PlacementScore - *out = new(float64) - **out = **in + if in.Placement != nil { + in, out := &in.Placement, &out.Placement + *out = new(PlacementStatus) + (*in).DeepCopyInto(*out) } } @@ -1131,6 +1131,26 @@ func (in *ParetoConfig) DeepCopy() *ParetoConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlacementStatus) DeepCopyInto(out *PlacementStatus) { + *out = *in + if in.Score != nil { + in, out := &in.Score, &out.Score + *out = new(float64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlacementStatus. +func (in *PlacementStatus) DeepCopy() *PlacementStatus { + if in == nil { + return nil + } + out := new(PlacementStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProfilingResultsStatus) DeepCopyInto(out *ProfilingResultsStatus) { *out = *in diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml index d92617d51af6..c8158442a69a 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeployments.yaml @@ -11947,27 +11947,27 @@ spec: description: ObservedGeneration is the most recent generation observed by the controller. format: int64 type: integer - placementScore: + placement: description: |- - PlacementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation uses - the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - placementScoreState: - description: |- - PlacementScoreState indicates placement score reporting state. See - PlacementScoreState for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string + Placement groups DGD-level scheduler placement signals (score, reporting + state, and any future placement fields). + properties: + score: + description: |- + Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0] + where higher is better and 1.0 is the best possible placement. + maximum: 1 + minimum: 0 + type: number + state: + description: State indicates placement score reporting state. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object restart: description: Restart contains the status of the restart of the graph deployment. properties: @@ -21334,27 +21334,33 @@ spec: description: observedGeneration is the most recent generation observed by the controller. format: int64 type: integer - placementScore: - description: |- - placementScore is the DGD-level scheduler placement score aggregated from - relevant scheduler placement units. Normalized to [0.0, 1.0] where higher - is better and 1.0 represents the best possible placement. Aggregation uses - the minimum across placement units so the value is a worst-placement - signal for the graph. Scores are only comparable across DGDs that share - the same scheduler scoring contract and version. - maximum: 1 - minimum: 0 - type: number - placementScoreState: + placement: description: |- - placementScoreState indicates placement score reporting state. See - PlacementScoreState for the semantics of each value. - enum: - - Reported - - Partial - - Unsupported - - Unknown - type: string + placement groups DGD-level scheduler placement signals (score, reporting + state, and any future placement fields). + properties: + score: + description: |- + score is the DGD-level scheduler placement score aggregated from + relevant scheduler placement units. Normalized to [0.0, 1.0] where higher + is better and 1.0 represents the best possible placement. Aggregation + uses the minimum across placement units so the value is a worst-placement + signal for the graph. Scores are only comparable across DGDs that share + the same scheduler scoring contract and version. + maximum: 1 + minimum: 0 + type: number + state: + description: |- + state indicates placement score reporting state. See PlacementScoreState + for the semantics of each value. + enum: + - Reported + - Partial + - Unsupported + - Unknown + type: string + type: object restart: description: restart contains the status of a graph-level restart. properties: diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 4d8014e6409c..3b7ccf997c71 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -25,7 +25,6 @@ import ( groveconstants "github.com/ai-dynamo/grove/operator/api/common/constants" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" - "github.com/go-logr/logr" "github.com/imdario/mergo" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" @@ -290,7 +289,7 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr // Compute DGD-level placement score from the Grove PodCliqueSet when on // the Grove pathway. Score is sourced from scheduler PodGang statuses and // is purely informational — it does not affect readiness. - r.updatePlacementScoreStatus(ctx, logger, dynamoDeployment) + r.updatePlacementStatus(ctx, dynamoDeployment) if err != nil { logger.Error(err, "failed to reconcile the resources") @@ -696,33 +695,37 @@ func (r *DynamoGraphDeploymentReconciler) getExistingGrovePodCliqueSet(ctx conte return pcs, nil } -// updatePlacementScoreStatus writes the placement score fields on every -// reconcile so status is never left stale. Per the placement-score DEP, -// every backend must set placementScoreState after the first reconciliation: +// updatePlacementStatus writes the placement status on every reconcile so +// status is never left stale. Per the placement-score DEP, every backend must +// set Placement.State after the first reconciliation: // - non-Grove pathways report Unsupported (no placement score source), // - Grove read failures report Unknown (indeterminate; may recover), // - otherwise the aggregator's classification is written through. // -// In every non-Reported/Partial branch PlacementScore is cleared so a value +// In every non-Reported/Partial branch Placement.Score is cleared so a value // from a previous reconcile cannot linger. -func (r *DynamoGraphDeploymentReconciler) updatePlacementScoreStatus(ctx context.Context, logger logr.Logger, dgd *nvidiacomv1beta1.DynamoGraphDeployment) { +func (r *DynamoGraphDeploymentReconciler) updatePlacementStatus(ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment) { if !r.isGrovePathway(dgd) { - dgd.Status.PlacementScore = nil - dgd.Status.PlacementScoreState = nvidiacomv1beta1.PlacementScoreStateUnsupported + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{ + State: nvidiacomv1beta1.PlacementScoreStateUnsupported, + } return } pcs, pcsErr := r.getExistingGrovePodCliqueSet(ctx, dgd) if pcsErr != nil { - logger.V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) - dgd.Status.PlacementScore = nil - dgd.Status.PlacementScoreState = nvidiacomv1beta1.PlacementScoreStateUnknown + log.FromContext(ctx).V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{ + State: nvidiacomv1beta1.PlacementScoreStateUnknown, + } return } score, state := dynamo.AggregatePlacementScore(pcs) - dgd.Status.PlacementScore = score - dgd.Status.PlacementScoreState = state + dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{ + Score: score, + State: state, + } } func restartAnnotationsFromPodCliqueSet(pcs *grovev1alpha1.PodCliqueSet) map[string]string { diff --git a/docs/kubernetes/api-reference.md b/docs/kubernetes/api-reference.md index 930fc36592ca..bfbe3911f4c2 100644 --- a/docs/kubernetes/api-reference.md +++ b/docs/kubernetes/api-reference.md @@ -737,8 +737,7 @@ _Appears in:_ | `restart` _[RestartStatus](#restartstatus)_ | Restart contains the status of the restart of the graph deployment. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ServiceCheckpointStatus](#servicecheckpointstatus))_ | Checkpoints contains per-service checkpoint status information.
The map key is the service name from spec.services. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | RollingUpdate tracks the progress of operator manged rolling updates.
Currently only supported for singl-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| -| `placementScore` _float_ | PlacementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation uses
the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| -| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | PlacementScoreState indicates placement score reporting state. See
PlacementScoreState for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| +| `placement` _[PlacementStatus](#placementstatus)_ | Placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| #### DynamoModel @@ -1140,7 +1139,7 @@ _Validation:_ - Enum: [Reported Partial Unsupported Unknown] _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [PlacementStatus](#placementstatus) | Field | Description | | --- | --- | @@ -1150,6 +1149,25 @@ _Appears in:_ | `Unknown` | | +#### PlacementStatus + + + +PlacementStatus groups DGD-level scheduler placement fields under a single +status object so future placement signals can be added without a schema +break. See the v1beta1 PlacementStatus for the authoritative field docs. + + + +_Appears in:_ +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `score` _float_ | Score is the DGD-level scheduler placement score. Normalized to [0.0, 1.0]
where higher is better and 1.0 is the best possible placement. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +| `state` _[PlacementScoreState](#placementscorestate)_ | State indicates placement score reporting state. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| + + #### PodReference @@ -2395,8 +2413,7 @@ _Appears in:_ | `restart` _[RestartStatus](#restartstatus)_ | restart contains the status of a graph-level restart. | | Optional: \{\}
| | `checkpoints` _object (keys:string, values:[ComponentCheckpointStatus](#componentcheckpointstatus))_ | checkpoints contains per-component checkpoint status, keyed by component name. | | Optional: \{\}
| | `rollingUpdate` _[RollingUpdateStatus](#rollingupdatestatus)_ | rollingUpdate tracks the progress of operator-managed rolling updates.
Currently only supported for single-node, non-Grove deployments (DCD/Deployment). | | Optional: \{\}
| -| `placementScore` _float_ | placementScore is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation uses
the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| -| `placementScoreState` _[PlacementScoreState](#placementscorestate)_ | placementScoreState indicates placement score reporting state. See
PlacementScoreState for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| +| `placement` _[PlacementStatus](#placementstatus)_ | placement groups DGD-level scheduler placement signals (score, reporting
state, and any future placement fields). | | Optional: \{\}
| #### EPPConfig @@ -2775,14 +2792,14 @@ Every backend must set this field after the first reconciliation: - Partial: a score is available for some but not all placement units. - Unsupported: the backend does not surface a placement score at all. - Unknown: the backend supports scores but the current value is - indeterminate (e.g. read failure, not yet populated by the - scheduler). When set, PlacementScore must be cleared. + indeterminate (e.g. read failure, not yet populated by the + scheduler). When set, PlacementStatus.Score must be cleared. _Validation:_ - Enum: [Reported Partial Unsupported Unknown] _Appears in:_ -- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) +- [PlacementStatus](#placementstatus) | Field | Description | | --- | --- | @@ -2792,6 +2809,25 @@ _Appears in:_ | `Unknown` | | +#### PlacementStatus + + + +PlacementStatus groups DGD-level scheduler placement fields under a single +status object so future placement signals (e.g. scheduler contract version, +last-report timestamp, per-unit reports) can be added without a schema break. + + + +_Appears in:_ +- [DynamoGraphDeploymentStatus](#dynamographdeploymentstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `score` _float_ | score is the DGD-level scheduler placement score aggregated from
relevant scheduler placement units. Normalized to [0.0, 1.0] where higher
is better and 1.0 represents the best possible placement. Aggregation
uses the minimum across placement units so the value is a worst-placement
signal for the graph. Scores are only comparable across DGDs that share
the same scheduler scoring contract and version. | | Maximum: 1
Minimum: 0
Optional: \{\}
| +| `state` _[PlacementScoreState](#placementscorestate)_ | state indicates placement score reporting state. See PlacementScoreState
for the semantics of each value. | | Enum: [Reported Partial Unsupported Unknown]
Optional: \{\}
| + + #### ProfilingPhase _Underlying type:_ _string_ From 5214cf81588ffba64746ade56b89102e0c7a8017 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 Jul 2026 11:27:50 -0400 Subject: [PATCH 6/8] test(operator): exclude PlacementStatus from cross-field fuzz filler Register no-op custom fuzzers for the v1alpha1 and v1beta1 PlacementStatus so the round-trip fuzz filler does not draw random bits for Placement. Placement round-trip is already covered by TestDGD_RoundTrip_Status. Signed-off-by: ashnamehrotra --- deploy/operator/api/roundtrip_fuzz_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index 83603f3559d3..37c20c56ead0 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -216,6 +216,12 @@ func dynamoFuzzerFuncs(_ runtimeserializer.CodecFactory) []any { fuzzAlphaDGDRStatus, fuzzBetaDGDRSpec, fuzzBetaDGDRStatus, + // PlacementStatus (v1alpha1 + v1beta1): keep the fuzz filler from + // drawing random bits for this pointer field. Placement conversion + // is exercised by TestDGD_RoundTrip_Status; excluding it here keeps + // the fuzz RNG stream stable across schema growth on DGDStatus. + func(p *v1beta1.PlacementStatus, _ randfill.Continue) {}, + func(p *v1alpha1.PlacementStatus, _ randfill.Continue) {}, // v1beta1 Components: the listMapKey marker requires name // to be non-empty and unique; MaxItems caps the length at 25. // Enforce both so the input is admissible. From 36d2d2794aa46afcc519fbe3217a164fe7bb0b52 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Thu, 16 Jul 2026 10:33:15 -0400 Subject: [PATCH 7/8] refactor(operator): land PlacementStatus schema-only Address @sttts P1 on grove.go:300. AggregatePlacementScore returned Unsupported for every non-empty PodGang list, so Reported/Partial and status.placement.score were unreachable until Grove exposes a placement score. Until the DEP #10064 open question on score source is resolved, land the API shape (types, CRD, conversion, round-trip tests) without wiring the controller. Removed: - dynamo.AggregatePlacementScore and its unit test - controller.updatePlacementStatus and its call site in Reconcile Kept: - PlacementStatus struct with Score + State (v1beta1 + v1alpha1) - Enum with Reported / Partial / Unsupported / Unknown - Score CRD bounds [0, 1] - v1alpha1<->v1beta1 conversion + round-trip test Doc-comment on v1beta1 PlacementStatus notes the schema-only state and points at DEP #10064. Regenerated CRDs, api-reference, and helm CRD copy. Signed-off-by: ashnamehrotra --- deploy/operator/api/v1beta1/common.go | 5 ++ .../dynamographdeployment_controller.go | 38 -------------- deploy/operator/internal/dynamo/grove.go | 34 ------------- deploy/operator/internal/dynamo/grove_test.go | 49 ------------------- docs/kubernetes/api-reference.md | 5 ++ 5 files changed, 10 insertions(+), 121 deletions(-) diff --git a/deploy/operator/api/v1beta1/common.go b/deploy/operator/api/v1beta1/common.go index e4ab79c6d9a8..0f3745f62342 100644 --- a/deploy/operator/api/v1beta1/common.go +++ b/deploy/operator/api/v1beta1/common.go @@ -588,6 +588,11 @@ const ( // PlacementStatus groups DGD-level scheduler placement fields under a single // status object so future placement signals (e.g. scheduler contract version, // last-report timestamp, per-unit reports) can be added without a schema break. +// +// The score source is an open question in DEP #10064 (Grove mirror, typed Grove +// scheduler API, or unstructured provider). Until a source is selected and +// implemented, the DGD controller does not write this field; the schema and +// conversion are landed here so downstream consumers can rely on the shape. type PlacementStatus struct { // score is the DGD-level scheduler placement score aggregated from // relevant scheduler placement units. Normalized to [0.0, 1.0] where higher diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 7e4db0490ca2..3a28963b8177 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -296,11 +296,6 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr dynamoDeployment.Status.Components = reconcileResult.ComponentStatus dynamoDeployment.Status.Restart = reconcileResult.RestartStatus - // Compute DGD-level placement score from the Grove PodCliqueSet when on - // the Grove pathway. Score is sourced from scheduler PodGang statuses and - // is purely informational — it does not affect readiness. - r.updatePlacementStatus(ctx, dynamoDeployment) - // Override state based on rolling update status if a rolling update is in progress if dynamoDeployment.Status.RollingUpdate != nil { switch dynamoDeployment.Status.RollingUpdate.Phase { @@ -709,39 +704,6 @@ func (r *DynamoGraphDeploymentReconciler) getExistingGrovePodCliqueSet(ctx conte return pcs, nil } -// updatePlacementStatus writes the placement status on every reconcile so -// status is never left stale. Per the placement-score DEP, every backend must -// set Placement.State after the first reconciliation: -// - non-Grove pathways report Unsupported (no placement score source), -// - Grove read failures report Unknown (indeterminate; may recover), -// - otherwise the aggregator's classification is written through. -// -// In every non-Reported/Partial branch Placement.Score is cleared so a value -// from a previous reconcile cannot linger. -func (r *DynamoGraphDeploymentReconciler) updatePlacementStatus(ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment) { - if !r.isGrovePathway(dgd) { - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{ - State: nvidiacomv1beta1.PlacementScoreStateUnsupported, - } - return - } - - pcs, pcsErr := r.getExistingGrovePodCliqueSet(ctx, dgd) - if pcsErr != nil { - log.FromContext(ctx).V(1).Info("Failed to get PodCliqueSet for placement score", "error", pcsErr) - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{ - State: nvidiacomv1beta1.PlacementScoreStateUnknown, - } - return - } - - score, state := dynamo.AggregatePlacementScore(pcs) - dgd.Status.Placement = &nvidiacomv1beta1.PlacementStatus{ - Score: score, - State: state, - } -} - func restartAnnotationsFromPodCliqueSet(pcs *grovev1alpha1.PodCliqueSet) map[string]string { restartAnnotations := make(map[string]string) if pcs == nil { diff --git a/deploy/operator/internal/dynamo/grove.go b/deploy/operator/internal/dynamo/grove.go index 876f985995dc..3d51f790b04f 100644 --- a/deploy/operator/internal/dynamo/grove.go +++ b/deploy/operator/internal/dynamo/grove.go @@ -266,40 +266,6 @@ func CheckPCSGReady(ctx context.Context, client client.Client, resourceName, nam return true, "", serviceStatus, nil } -// AggregatePlacementScore reads PodGangStatutes from the given PodCliqueSet -// and derives the DGD-level placement score. This implements the "Grove -// mirror" placement-score source alternative; the source of truth is still -// under discussion (see the placement-score DEP Open Questions). -// -// When multiple PodGangs report a score the minimum is used: the DGD-level -// value is a worst-placement signal, matching the DEP's rationale that one -// poorly placed serving unit should not be hidden by a weighted average. -// -// Absence is non-failing: -// - nil PodCliqueSet: Unsupported (no Grove object to read from). -// - PodGangStatutes empty: Unknown (Grove hasn't populated the status yet, -// which is indeterminate rather than a permanent absence). -// -// NOTE: The current Grove alpha.8 PodGangStatus has no PlacementScore field. -// Until Grove exposes it, this function reports Unsupported when gangs are -// present. Replace the TODO below with the actual field read (and any -// partial-report handling) once available. -func AggregatePlacementScore(pcs *grovev1alpha1.PodCliqueSet) (score *float64, state v1beta1.PlacementScoreState) { - if pcs == nil { - return nil, v1beta1.PlacementScoreStateUnsupported - } - - gangs := pcs.Status.PodGangStatutes - if len(gangs) == 0 { - return nil, v1beta1.PlacementScoreStateUnknown - } - - // TODO: read gang.PlacementScore once Grove PodGangStatus exposes the - // field. Until then, report Unsupported so absence is explicit and - // non-failing. - return nil, v1beta1.PlacementScoreStateUnsupported -} - // specToGroveTopologyConstraint converts a deployment-level topology constraint // to the current Grove API shape. func specToGroveTopologyConstraint(tc *v1beta1.SpecTopologyConstraint) *grovev1alpha1.TopologyConstraint { diff --git a/deploy/operator/internal/dynamo/grove_test.go b/deploy/operator/internal/dynamo/grove_test.go index ea7927f3209b..c29c99f47cca 100644 --- a/deploy/operator/internal/dynamo/grove_test.go +++ b/deploy/operator/internal/dynamo/grove_test.go @@ -1430,52 +1430,3 @@ func Test_GetComponentReadinessAndServiceReplicaStatuses(t *testing.T) { }) } } - -func TestAggregatePlacementScore_CurrentBehavior(t *testing.T) { - tests := []struct { - name string - pcs *grovev1alpha1.PodCliqueSet - wantScore *float64 - wantState v1beta1.PlacementScoreState - }{ - { - name: "nil pod clique set", - pcs: nil, - wantScore: nil, - wantState: v1beta1.PlacementScoreStateUnsupported, - }, - { - name: "no pod gangs reported", - pcs: &grovev1alpha1.PodCliqueSet{ - Status: grovev1alpha1.PodCliqueSetStatus{}, - }, - wantScore: nil, - wantState: v1beta1.PlacementScoreStateUnknown, - }, - { - name: "pod gangs present but no score field in current grove api", - pcs: &grovev1alpha1.PodCliqueSet{ - Status: grovev1alpha1.PodCliqueSetStatus{ - PodGangStatutes: []grovev1alpha1.PodGangStatus{{ - Name: "pg-0", - Phase: grovev1alpha1.PodGangRunning, - }}, - }, - }, - wantScore: nil, - wantState: v1beta1.PlacementScoreStateUnsupported, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - score, state := AggregatePlacementScore(tt.pcs) - if score != nil || tt.wantScore != nil { - t.Fatalf("score = %v, want %v", score, tt.wantScore) - } - if state != tt.wantState { - t.Fatalf("state = %q, want %q", state, tt.wantState) - } - }) - } -} diff --git a/docs/kubernetes/api-reference.md b/docs/kubernetes/api-reference.md index c50a4d343e47..a200247f7a43 100644 --- a/docs/kubernetes/api-reference.md +++ b/docs/kubernetes/api-reference.md @@ -2818,6 +2818,11 @@ PlacementStatus groups DGD-level scheduler placement fields under a single status object so future placement signals (e.g. scheduler contract version, last-report timestamp, per-unit reports) can be added without a schema break. +The score source is an open question in DEP #10064 (Grove mirror, typed Grove +scheduler API, or unstructured provider). Until a source is selected and +implemented, the DGD controller does not write this field; the schema and +conversion are landed here so downstream consumers can rely on the shape. + _Appears in:_ From effa9fa17d595282ae0326c2ca4d66423df57061 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Thu, 16 Jul 2026 11:05:27 -0400 Subject: [PATCH 8/8] test(operator): fuzz PlacementStatus with admissible enum + score values Address @sttts on roundtrip_fuzz_test.go:224. Replace the no-op PlacementStatus fuzzers with real randomizers that pick State from the four PlacementScoreState enum values and Score from either nil or a value inside the CRD [0, 1] bounds. Add an oneOfPtr helper alongside oneOf for the sometimes-nil-sometimes-value pattern that optional API pointer fields need. Signed-off-by: ashnamehrotra --- deploy/operator/api/roundtrip_fuzz_test.go | 39 ++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index 37c20c56ead0..8d6dde66f9c7 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -216,12 +216,28 @@ func dynamoFuzzerFuncs(_ runtimeserializer.CodecFactory) []any { fuzzAlphaDGDRStatus, fuzzBetaDGDRSpec, fuzzBetaDGDRStatus, - // PlacementStatus (v1alpha1 + v1beta1): keep the fuzz filler from - // drawing random bits for this pointer field. Placement conversion - // is exercised by TestDGD_RoundTrip_Status; excluding it here keeps - // the fuzz RNG stream stable across schema growth on DGDStatus. - func(p *v1beta1.PlacementStatus, _ randfill.Continue) {}, - func(p *v1alpha1.PlacementStatus, _ randfill.Continue) {}, + // PlacementStatus (v1alpha1 + v1beta1): pick admissible values so the + // round-trip fuzzer exercises Placement without producing shapes the CRD + // schema rejects. State draws from the enum; Score draws either nil or a + // value inside the [0, 1] bounds enforced by the CRD. + func(p *v1beta1.PlacementStatus, c randfill.Continue) { + p.State = oneOf(c, + v1beta1.PlacementScoreStateReported, + v1beta1.PlacementScoreStatePartial, + v1beta1.PlacementScoreStateUnsupported, + v1beta1.PlacementScoreStateUnknown, + ) + p.Score = oneOfPtr(c, 0.0, 0.25, 0.5, 0.75, 1.0) + }, + func(p *v1alpha1.PlacementStatus, c randfill.Continue) { + p.State = oneOf(c, + v1alpha1.PlacementScoreStateReported, + v1alpha1.PlacementScoreStatePartial, + v1alpha1.PlacementScoreStateUnsupported, + v1alpha1.PlacementScoreStateUnknown, + ) + p.Score = oneOfPtr(c, 0.0, 0.25, 0.5, 0.75, 1.0) + }, // v1beta1 Components: the listMapKey marker requires name // to be non-empty and unique; MaxItems caps the length at 25. // Enforce both so the input is admissible. @@ -351,6 +367,17 @@ func oneOf[T any](c randfill.Continue, values ...T) T { return values[c.Intn(len(values))] } +// oneOfPtr returns nil roughly half the time; otherwise a pointer to one of +// values. Useful for optional API fields that must either be unset or draw +// from a constrained value set. +func oneOfPtr[T any](c randfill.Continue, values ...T) *T { + if c.Bool() { + return nil + } + v := oneOf(c, values...) + return &v +} + func fuzzJSONValue(c randfill.Continue, depth int) any { if depth >= 2 { switch c.Uint32() % 5 {