Skip to content

Commit 962ff6b

Browse files
Add WorkerPool scheduling fields (agent-substrate#247)
**Updated Original PR Description** ## Summary This PR adds a small, controlled set of scheduling fields to WorkerPool so WorkerPool-managed Pods can be placed onto appropriate Kubernetes nodes. Added fields: - `nodeSelector` - `tolerations` - `priorityClassName` - `nodeAffinity` ## Scope This intentionally does not expose a full `PodTemplateSpec`, and does not add support for full `affinity`, `podAffinity`, or `podAntiAffinity`. Resource requests/limits are intentionally left out of scope for this CL while the WorkerPool resource model is still being discussed separately. The goal is to keep the first version small while addressing the WorkerPool-to-Kubernetes-Node scheduling gap discussed in [agent-substrate#212](agent-substrate#212). `nodeAffinity` is included to support heterogeneous node pools where `nodeSelector` is too limited, such as requiring or preferring one of several equivalent accelerator/local-SSD/cache node pools, or expressing soft preferences during node pool migration. `priorityClassName` is included to help preserve warm WorkerPool capacity under cluster pressure. Since WorkerPool pods are the slots available for actor resume, higher-priority pools can protect latency-sensitive or interactive actor workloads from displacement by lower-priority batch workloads, and allow different WorkerPools to represent different service classes, e.g. interactive vs. batch. ## Related issue Part of agent-substrate#212. Related to agent-substrate#47.
1 parent a1ec7e3 commit 962ff6b

10 files changed

Lines changed: 1042 additions & 60 deletions

File tree

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package controllers
16+
17+
import (
18+
corev1 "k8s.io/api/core/v1"
19+
appsv1ac "k8s.io/client-go/applyconfigurations/apps/v1"
20+
corev1ac "k8s.io/client-go/applyconfigurations/core/v1"
21+
metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
22+
23+
"github.com/agent-substrate/substrate/internal/ateompath"
24+
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
25+
)
26+
27+
// buildDeploymentApplyConfig constructs the SSA apply configuration for the
28+
// Deployment managed by a WorkerPool. Only fields owned by this controller
29+
// are declared here.
30+
func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.DeploymentApplyConfiguration {
31+
containerAC := corev1ac.Container().
32+
WithName("ateom").
33+
WithImage(wp.Spec.AteomImage).
34+
WithArgs(
35+
"--pod-uid=$(POD_UID)",
36+
).
37+
WithSecurityContext(corev1ac.SecurityContext().
38+
WithPrivileged(true).
39+
WithRunAsUser(0).
40+
WithRunAsGroup(0)).
41+
WithEnv(
42+
corev1ac.EnvVar().
43+
WithName("POD_UID").
44+
WithValueFrom(corev1ac.EnvVarSource().
45+
WithFieldRef(corev1ac.ObjectFieldSelector().
46+
WithFieldPath("metadata.uid"))),
47+
).
48+
WithVolumeMounts(corev1ac.VolumeMount().
49+
WithName("run-ateom").
50+
WithMountPath(ateompath.BasePath))
51+
52+
podSpecAC := corev1ac.PodSpec().
53+
WithSecurityContext(corev1ac.PodSecurityContext().
54+
WithRunAsUser(0).
55+
WithRunAsGroup(0)).
56+
WithVolumes(corev1ac.Volume().
57+
WithName("run-ateom").
58+
WithHostPath(corev1ac.HostPathVolumeSource().
59+
WithPath(ateompath.BasePath).
60+
WithType(corev1.HostPathDirectoryOrCreate)))
61+
62+
applyWorkerPoolPodTemplate(podSpecAC, wp.Spec.Template)
63+
podSpecAC.WithContainers(containerAC)
64+
65+
return appsv1ac.Deployment(deploymentName(wp.Name), wp.Namespace).
66+
WithOwnerReferences(metav1ac.OwnerReference().
67+
WithAPIVersion(atev1alpha1.GroupVersion.String()).
68+
WithKind("WorkerPool").
69+
WithName(wp.Name).
70+
WithUID(wp.UID).
71+
WithController(true).
72+
WithBlockOwnerDeletion(true)).
73+
WithSpec(appsv1ac.DeploymentSpec().
74+
WithReplicas(wp.Spec.Replicas).
75+
WithSelector(metav1ac.LabelSelector().
76+
WithMatchLabels(map[string]string{"ate.dev/worker-pool": wp.Name})).
77+
WithTemplate(corev1ac.PodTemplateSpec().
78+
WithLabels(map[string]string{
79+
"ate.dev/worker-pool": wp.Name,
80+
}).
81+
WithSpec(podSpecAC)))
82+
}
83+
84+
func applyWorkerPoolPodTemplate(
85+
podSpecAC *corev1ac.PodSpecApplyConfiguration,
86+
tmpl *atev1alpha1.WorkerPoolPodTemplate,
87+
) {
88+
podSpecAC.NodeSelector = map[string]string{}
89+
podSpecAC.Tolerations = []corev1ac.TolerationApplyConfiguration{}
90+
podSpecAC.WithPriorityClassName("")
91+
podSpecAC.WithAffinity(corev1ac.Affinity())
92+
93+
if tmpl == nil {
94+
return
95+
}
96+
97+
if tmpl.NodeSelector != nil {
98+
podSpecAC.WithNodeSelector(tmpl.NodeSelector)
99+
}
100+
podSpecAC.Tolerations = tolerationApplyValues(tolerationsToApply(tmpl.Tolerations))
101+
podSpecAC.WithPriorityClassName(tmpl.PriorityClassName)
102+
103+
if tmpl.NodeAffinity != nil {
104+
podSpecAC.WithAffinity(corev1ac.Affinity().WithNodeAffinity(nodeAffinityToApply(tmpl.NodeAffinity)))
105+
}
106+
}
107+
108+
func tolerationApplyValues(tolerations []*corev1ac.TolerationApplyConfiguration) []corev1ac.TolerationApplyConfiguration {
109+
out := make([]corev1ac.TolerationApplyConfiguration, 0, len(tolerations))
110+
for _, toleration := range tolerations {
111+
out = append(out, *toleration)
112+
}
113+
return out
114+
}
115+
116+
func tolerationsToApply(tolerations []corev1.Toleration) []*corev1ac.TolerationApplyConfiguration {
117+
out := make([]*corev1ac.TolerationApplyConfiguration, 0, len(tolerations))
118+
for i := range tolerations {
119+
t := &tolerations[i]
120+
ac := corev1ac.Toleration()
121+
if t.Key != "" {
122+
ac.WithKey(t.Key)
123+
}
124+
if t.Operator != "" {
125+
ac.WithOperator(t.Operator)
126+
}
127+
if t.Value != "" {
128+
ac.WithValue(t.Value)
129+
}
130+
if t.Effect != "" {
131+
ac.WithEffect(t.Effect)
132+
}
133+
if t.TolerationSeconds != nil {
134+
ac.WithTolerationSeconds(*t.TolerationSeconds)
135+
}
136+
out = append(out, ac)
137+
}
138+
return out
139+
}
140+
141+
func nodeAffinityToApply(na *corev1.NodeAffinity) *corev1ac.NodeAffinityApplyConfiguration {
142+
ac := corev1ac.NodeAffinity()
143+
if na.RequiredDuringSchedulingIgnoredDuringExecution != nil {
144+
ac.WithRequiredDuringSchedulingIgnoredDuringExecution(nodeSelectorToApply(na.RequiredDuringSchedulingIgnoredDuringExecution))
145+
}
146+
for i := range na.PreferredDuringSchedulingIgnoredDuringExecution {
147+
term := &na.PreferredDuringSchedulingIgnoredDuringExecution[i]
148+
ac.WithPreferredDuringSchedulingIgnoredDuringExecution(preferredSchedulingTermToApply(term))
149+
}
150+
return ac
151+
}
152+
153+
func nodeSelectorToApply(ns *corev1.NodeSelector) *corev1ac.NodeSelectorApplyConfiguration {
154+
ac := corev1ac.NodeSelector()
155+
for i := range ns.NodeSelectorTerms {
156+
ac.WithNodeSelectorTerms(nodeSelectorTermToApply(&ns.NodeSelectorTerms[i]))
157+
}
158+
return ac
159+
}
160+
161+
func preferredSchedulingTermToApply(term *corev1.PreferredSchedulingTerm) *corev1ac.PreferredSchedulingTermApplyConfiguration {
162+
return corev1ac.PreferredSchedulingTerm().
163+
WithWeight(term.Weight).
164+
WithPreference(nodeSelectorTermToApply(&term.Preference))
165+
}
166+
167+
func nodeSelectorTermToApply(term *corev1.NodeSelectorTerm) *corev1ac.NodeSelectorTermApplyConfiguration {
168+
ac := corev1ac.NodeSelectorTerm()
169+
for i := range term.MatchExpressions {
170+
ac.WithMatchExpressions(nodeSelectorRequirementToApply(&term.MatchExpressions[i]))
171+
}
172+
for i := range term.MatchFields {
173+
ac.WithMatchFields(nodeSelectorRequirementToApply(&term.MatchFields[i]))
174+
}
175+
return ac
176+
}
177+
178+
func nodeSelectorRequirementToApply(req *corev1.NodeSelectorRequirement) *corev1ac.NodeSelectorRequirementApplyConfiguration {
179+
ac := corev1ac.NodeSelectorRequirement().WithKey(req.Key).WithOperator(req.Operator)
180+
if len(req.Values) > 0 {
181+
ac.WithValues(req.Values...)
182+
}
183+
return ac
184+
}

0 commit comments

Comments
 (0)