Skip to content

Commit 48b8e69

Browse files
mahmoudk1000mahmoudk1000
authored
[FEAT] add support for customiziable runtimeClass & podAnnotaions (#958)
* [FEAT] Add runtimeClass for vllmRuntime Signed-off-by: Mahmoud Ayman <mahmoudk1000@gmail.com> * [FEAT] Add podAnnotaions for vllmRuntime Signed-off-by: Mahmoud Ayman <mahmoudk1000@gmail.com> --------- Signed-off-by: Mahmoud Ayman <mahmoudk1000@gmail.com>
1 parent 8909ed7 commit 48b8e69

4 files changed

Lines changed: 114 additions & 16 deletions

File tree

‎operator/api/v1alpha1/vllmruntime_types.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,19 @@ type DeploymentConfig struct {
4141
// +kubebuilder:default=RollingUpdate
4242
DeployStrategy string `json:"deploymentStrategy,omitempty"`
4343

44+
// RuntimeClass
45+
// +kubebuilder:default=nividia
46+
RuntimeClass string `json:"runtimeClass,omitempty"`
47+
4448
// Resource requirements
4549
Resources ResourceRequirements `json:"resources"`
4650

4751
// Image configuration
4852
Image ImageSpec `json:"image"`
4953

54+
// Pod annotations
55+
PodAnnotations map[string]string `json:"podAnnotations,omitempty"`
56+
5057
// Sidecar configuration
5158
SidecarConfig SidecarConfig `json:"sidecarConfig,omitempty"`
5259
}

‎operator/api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎operator/config/crd/bases/production-stack.vllm.ai_vllmruntimes.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,11 @@ spec:
257257
type: object
258258
x-kubernetes-map-type: atomic
259259
type: array
260+
podAnnotations:
261+
additionalProperties:
262+
type: string
263+
description: Pod annotations
264+
type: object
260265
replicas:
261266
default: 1
262267
description: Replicas
@@ -274,6 +279,10 @@ spec:
274279
memory:
275280
type: string
276281
type: object
282+
runtimeClass:
283+
default: nividia
284+
description: RuntimeClass
285+
type: string
277286
sidecarConfig:
278287
description: Sidecar configuration
279288
properties:

‎operator/internal/controller/vllmruntime_controller.go‎

Lines changed: 91 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -332,12 +332,26 @@ func (r *VLLMRuntimeReconciler) Reconcile(
332332
if *cfg.MinReplicas > cfg.MaxReplicas {
333333
log.Error(nil, "Invalid autoscaling config: minReplicas must be <= maxReplicas",
334334
"minReplicas", *cfg.MinReplicas, "maxReplicas", cfg.MaxReplicas)
335-
return ctrl.Result{}, fmt.Errorf("minReplicas (%d) must be <= maxReplicas (%d)", *cfg.MinReplicas, cfg.MaxReplicas)
335+
return ctrl.Result{}, fmt.Errorf(
336+
"minReplicas (%d) must be <= maxReplicas (%d)",
337+
*cfg.MinReplicas,
338+
cfg.MaxReplicas,
339+
)
336340
}
337341
if cfg.MaxReplicas < vllmRuntime.Spec.DeploymentConfig.Replicas {
338-
log.Error(nil, "Invalid autoscaling config: maxReplicas must be >= deploymentConfig.replicas",
339-
"maxReplicas", cfg.MaxReplicas, "replicas", vllmRuntime.Spec.DeploymentConfig.Replicas)
340-
return ctrl.Result{}, fmt.Errorf("maxReplicas (%d) must be >= deploymentConfig.replicas (%d)", cfg.MaxReplicas, vllmRuntime.Spec.DeploymentConfig.Replicas)
342+
log.Error(
343+
nil,
344+
"Invalid autoscaling config: maxReplicas must be >= deploymentConfig.replicas",
345+
"maxReplicas",
346+
cfg.MaxReplicas,
347+
"replicas",
348+
vllmRuntime.Spec.DeploymentConfig.Replicas,
349+
)
350+
return ctrl.Result{}, fmt.Errorf(
351+
"maxReplicas (%d) must be >= deploymentConfig.replicas (%d)",
352+
cfg.MaxReplicas,
353+
vllmRuntime.Spec.DeploymentConfig.Replicas,
354+
)
341355
}
342356
if err := r.reconcileScaledObject(ctx, vllmRuntime); err != nil {
343357
log.Error(err, "Failed to reconcile ScaledObject")
@@ -772,9 +786,11 @@ func (r *VLLMRuntimeReconciler) deploymentForVLLMRuntime(
772786
},
773787
Template: corev1.PodTemplateSpec{
774788
ObjectMeta: metav1.ObjectMeta{
775-
Labels: labels,
789+
Labels: labels,
790+
Annotations: vllmRuntime.Spec.DeploymentConfig.PodAnnotations,
776791
},
777792
Spec: corev1.PodSpec{
793+
RuntimeClassName: &vllmRuntime.Spec.DeploymentConfig.RuntimeClass,
778794
Affinity: affinity,
779795
Tolerations: vllmRuntime.Spec.DeploymentConfig.Toleration,
780796
ImagePullSecrets: imagePullSecrets,
@@ -1031,6 +1047,39 @@ func (r *VLLMRuntimeReconciler) deploymentNeedsUpdate(
10311047
return true
10321048
}
10331049

1050+
expectedRuntimeClass := expectedDep.Spec.Template.Spec.RuntimeClassName
1051+
actualRuntimeClass := dep.Spec.Template.Spec.RuntimeClassName
1052+
1053+
if !reflect.DeepEqual(expectedRuntimeClass, actualRuntimeClass) {
1054+
log.Info(
1055+
"RuntimeClass mismatch",
1056+
"expected",
1057+
expectedRuntimeClass,
1058+
"actual",
1059+
actualRuntimeClass,
1060+
)
1061+
return true
1062+
}
1063+
1064+
expectedPodAnnotations := expectedDep.Spec.Template.Annotations
1065+
actualPodAnnotations := dep.Spec.Template.Annotations
1066+
1067+
for k, v := range expectedPodAnnotations {
1068+
if actualPodAnnotations[k] != v {
1069+
log.Info(
1070+
"Pod annotations mismatch",
1071+
"key",
1072+
k,
1073+
"expected",
1074+
v,
1075+
"actual",
1076+
actualPodAnnotations[k],
1077+
)
1078+
1079+
return true
1080+
}
1081+
}
1082+
10341083
return false
10351084
}
10361085

@@ -1119,13 +1168,21 @@ func (r *VLLMRuntimeReconciler) reconcileScaledObject(
11191168
"scaleUp": map[string]interface{}{
11201169
"stabilizationWindowSeconds": *cfg.ScaleUpPolicy.StabilizationWindowSeconds,
11211170
"policies": []map[string]interface{}{
1122-
{"type": "Pods", "value": *cfg.ScaleUpPolicy.PodValue, "periodSeconds": *cfg.ScaleUpPolicy.PeriodSeconds},
1171+
{
1172+
"type": "Pods",
1173+
"value": *cfg.ScaleUpPolicy.PodValue,
1174+
"periodSeconds": *cfg.ScaleUpPolicy.PeriodSeconds,
1175+
},
11231176
},
11241177
},
11251178
"scaleDown": map[string]interface{}{
11261179
"stabilizationWindowSeconds": *cfg.ScaleDownPolicy.StabilizationWindowSeconds,
11271180
"policies": []map[string]interface{}{
1128-
{"type": "Pods", "value": *cfg.ScaleDownPolicy.PodValue, "periodSeconds": *cfg.ScaleDownPolicy.PeriodSeconds},
1181+
{
1182+
"type": "Pods",
1183+
"value": *cfg.ScaleDownPolicy.PodValue,
1184+
"periodSeconds": *cfg.ScaleDownPolicy.PeriodSeconds,
1185+
},
11291186
},
11301187
},
11311188
},
@@ -1138,42 +1195,60 @@ func (r *VLLMRuntimeReconciler) reconcileScaledObject(
11381195
"metadata": map[string]string{
11391196
"serverAddress": prometheusAddr,
11401197
"metricName": "vllm_incoming_keepalive",
1141-
"query": fmt.Sprintf(`sum(rate(vllm:num_incoming_requests_total{namespace="%s", model="%s"}[2m]) > bool 0)`, vllmRuntime.Namespace, servedModelName),
1142-
"threshold": "1",
1198+
"query": fmt.Sprintf(
1199+
`sum(rate(vllm:num_incoming_requests_total{namespace="%s", model="%s"}[2m]) > bool 0)`,
1200+
vllmRuntime.Namespace,
1201+
servedModelName,
1202+
),
1203+
"threshold": "1",
11431204
},
11441205
},
11451206
{
11461207
"type": "prometheus",
11471208
"metadata": map[string]string{
11481209
"serverAddress": prometheusAddr,
11491210
"metricName": "vllm_requests_running",
1150-
"query": fmt.Sprintf(`sum(vllm:num_requests_running{job="%s"})`, jobName),
1151-
"threshold": fmt.Sprintf("%d", *cfg.Triggers.RequestsRunningThreshold),
1211+
"query": fmt.Sprintf(
1212+
`sum(vllm:num_requests_running{job="%s"})`,
1213+
jobName,
1214+
),
1215+
"threshold": fmt.Sprintf("%d", *cfg.Triggers.RequestsRunningThreshold),
11521216
},
11531217
},
11541218
{
11551219
"type": "prometheus",
11561220
"metadata": map[string]string{
11571221
"serverAddress": prometheusAddr,
11581222
"metricName": "vllm_generation_tokens_rate",
1159-
"query": fmt.Sprintf(`sum(rate(vllm:generation_tokens_total{job="%s"}[1m]))`, jobName),
1160-
"threshold": fmt.Sprintf("%d", *cfg.Triggers.GenerationTokensThreshold),
1223+
"query": fmt.Sprintf(
1224+
`sum(rate(vllm:generation_tokens_total{job="%s"}[1m]))`,
1225+
jobName,
1226+
),
1227+
"threshold": fmt.Sprintf("%d", *cfg.Triggers.GenerationTokensThreshold),
11611228
},
11621229
},
11631230
{
11641231
"type": "prometheus",
11651232
"metadata": map[string]string{
11661233
"serverAddress": prometheusAddr,
11671234
"metricName": "vllm_prompt_tokens_rate",
1168-
"query": fmt.Sprintf(`sum(rate(vllm:prompt_tokens_total{job="%s"}[1m]))`, jobName),
1169-
"threshold": fmt.Sprintf("%d", *cfg.Triggers.PromptTokensThreshold),
1235+
"query": fmt.Sprintf(
1236+
`sum(rate(vllm:prompt_tokens_total{job="%s"}[1m]))`,
1237+
jobName,
1238+
),
1239+
"threshold": fmt.Sprintf("%d", *cfg.Triggers.PromptTokensThreshold),
11701240
},
11711241
},
11721242
},
11731243
}
11741244

11751245
scaledObject.Object["spec"] = spec
1176-
return r.Client.Patch(ctx, scaledObject, client.Apply, client.FieldOwner("vllmruntime-controller"))
1246+
return r.Client.Patch(
1247+
ctx,
1248+
scaledObject,
1249+
client.Apply,
1250+
client.FieldOwner("vllmruntime-controller"),
1251+
)
11771252
}
11781253

11791254
// serviceForVLLMRuntime returns a VLLMRuntime Service object

0 commit comments

Comments
 (0)