Skip to content

Commit 00a90a8

Browse files
BenTheElderBenTheElder
authored
Decouple Sandbox Binaries / Config from ActorTemplate (agent-substrate#261)
Fixes agent-substrate#253 > It's a good idea to open an issue first for discussion. - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR Should make something like agent-substrate#239 less awkward, and also agent-substrate#255 (we don't have to cram more fields into actortemplate for the shim binary and update _all_ of the demo actortemplates to include them).
1 parent 962ff6b commit 00a90a8

46 files changed

Lines changed: 1681 additions & 705 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmd/ateapi/internal/controlapi/functional_test.go‎

Lines changed: 71 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,8 @@ type testContext struct {
231231
fakeAtelet *FakeAteletServer
232232
cleanup func()
233233
actorTemplateLister listersv1alpha1.ActorTemplateLister
234+
workerPoolLister listersv1alpha1.WorkerPoolLister
235+
sandboxConfigLister listersv1alpha1.SandboxConfigLister
234236
}
235237

236238
// setupTest sets up a fully isolated test environment.
@@ -266,6 +268,8 @@ func setupTest(t *testing.T, ns string) *testContext {
266268

267269
substrateInformerFactory := externalversions.NewSharedInformerFactory(substrateClient, 0)
268270
actorTemplateLister := substrateInformerFactory.Api().V1alpha1().ActorTemplates().Lister()
271+
workerPoolLister := substrateInformerFactory.Api().V1alpha1().WorkerPools().Lister()
272+
sandboxConfigLister := substrateInformerFactory.Api().V1alpha1().SandboxConfigs().Lister()
269273

270274
ctx, cancel := context.WithCancel(context.Background())
271275

@@ -282,7 +286,7 @@ func setupTest(t *testing.T, ns string) *testContext {
282286

283287
// 4. Initialize Service
284288
dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer())
285-
service := NewService(persistence, actorTemplateLister, dialer, k8sClient)
289+
service := NewService(persistence, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient)
286290

287291
// 5. Start REAL gRPC Server for ATE API
288292
grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor))
@@ -343,6 +347,8 @@ func setupTest(t *testing.T, ns string) *testContext {
343347
fakeAtelet: fakeAtelet,
344348
cleanup: cleanup,
345349
actorTemplateLister: actorTemplateLister,
350+
workerPoolLister: workerPoolLister,
351+
sandboxConfigLister: sandboxConfigLister,
346352
}
347353
}
348354

@@ -363,18 +369,20 @@ func createTemplate(t *testing.T, tc *testContext, ns string) {
363369

364370
func createTemplateWithContainers(t *testing.T, tc *testContext, ns string, containers []atev1alpha1.Container) {
365371
t.Helper()
372+
373+
// Sandbox binaries now live on a (cluster-scoped) SandboxConfig resolved via
374+
// the actor's WorkerPool, not on the ActorTemplate. Create a default gvisor
375+
// SandboxConfig and the pool the template references so a boot-from-spec Run
376+
// can resolve its assets.
377+
ensureDefaultGvisorSandboxConfig(t, tc)
378+
ensureWorkerPool(t, tc, ns, "pool1")
379+
366380
actorTemplate := &atev1alpha1.ActorTemplate{
367381
ObjectMeta: metav1.ObjectMeta{
368382
Name: "tmpl1",
369383
Namespace: ns,
370384
},
371385
Spec: atev1alpha1.ActorTemplateSpec{
372-
Runsc: atev1alpha1.RunscConfig{
373-
AMD64: &atev1alpha1.RunscPlatformConfig{
374-
URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc",
375-
SHA256Hash: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63",
376-
},
377-
},
378386
PauseImage: "pause@sha256:abc",
379387
SnapshotsConfig: atev1alpha1.SnapshotsConfig{
380388
Location: "gs://fake-fake-fake",
@@ -413,6 +421,62 @@ func createTemplateWithContainers(t *testing.T, tc *testContext, ns string, cont
413421
}
414422
}
415423

424+
// ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor
425+
// SandboxConfig (idempotently) and waits for it to appear in the lister.
426+
func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) {
427+
t.Helper()
428+
const name = "gvisor-default"
429+
sc := &atev1alpha1.SandboxConfig{
430+
ObjectMeta: metav1.ObjectMeta{Name: name},
431+
Spec: atev1alpha1.SandboxConfigSpec{
432+
SandboxClass: atev1alpha1.SandboxClassGvisor,
433+
Default: true,
434+
Assets: map[string]map[string]atev1alpha1.AssetFile{
435+
"amd64": {"runsc": {
436+
URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc",
437+
SHA256: "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea842b63",
438+
}},
439+
"arm64": {"runsc": {
440+
URL: "gs://gvisor/releases/nightly/2026-05-19/aarch64/runsc",
441+
SHA256: "1ba2366ae2efceba166046f51a4104f9261c9cb72c6db8f5b3fe2dc57dea86b9",
442+
}},
443+
},
444+
},
445+
}
446+
if _, err := tc.substrateClient.ApiV1alpha1().SandboxConfigs().Create(context.Background(), sc, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
447+
t.Fatalf("failed to create default SandboxConfig: %v", err)
448+
}
449+
if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
450+
_, err := tc.sandboxConfigLister.Get(name)
451+
return err == nil, nil
452+
}); err != nil {
453+
t.Fatalf("default SandboxConfig not synced into lister: %v", err)
454+
}
455+
}
456+
457+
// ensureWorkerPool creates the namespaced WorkerPool an ActorTemplate references
458+
// (idempotently) and waits for it to appear in the lister.
459+
func ensureWorkerPool(t *testing.T, tc *testContext, ns, name string) {
460+
t.Helper()
461+
wp := &atev1alpha1.WorkerPool{
462+
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns},
463+
Spec: atev1alpha1.WorkerPoolSpec{
464+
Replicas: 1,
465+
AteomImage: "ateom@sha256:abc",
466+
SandboxClass: atev1alpha1.SandboxClassGvisor,
467+
},
468+
}
469+
if _, err := tc.substrateClient.ApiV1alpha1().WorkerPools(ns).Create(context.Background(), wp, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
470+
t.Fatalf("failed to create WorkerPool: %v", err)
471+
}
472+
if err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) {
473+
_, err := tc.workerPoolLister.WorkerPools(ns).Get(name)
474+
return err == nil, nil
475+
}); err != nil {
476+
t.Fatalf("WorkerPool not synced into lister: %v", err)
477+
}
478+
}
479+
416480
func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, nodeName string) {
417481
t.Helper()
418482
pod := &corev1.Pod{

‎cmd/ateapi/internal/controlapi/sandbox_assets.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package controlapi
16+
17+
import (
18+
"fmt"
19+
20+
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
21+
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
22+
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
23+
"k8s.io/apimachinery/pkg/labels"
24+
)
25+
26+
// resolveSandboxAssets determines the sandbox binaries an actor should boot with
27+
// and projects them onto the ateletpb.SandboxAssets atelet fetches. It resolves
28+
// the actor's WorkerPool, takes its SandboxClass (default gvisor), then picks the
29+
// SandboxConfig named by the pool — or, if none is named, the cluster default
30+
// SandboxConfig for that class.
31+
func resolveSandboxAssets(
32+
workerPoolLister listersv1alpha1.WorkerPoolLister,
33+
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
34+
at *atev1alpha1.ActorTemplate,
35+
) (*ateletpb.SandboxAssets, error) {
36+
ref := at.Spec.WorkerPoolRef
37+
wp, err := workerPoolLister.WorkerPools(ref.Namespace).Get(ref.Name)
38+
if err != nil {
39+
return nil, fmt.Errorf("while getting WorkerPool %s/%s: %w", ref.Namespace, ref.Name, err)
40+
}
41+
42+
class := wp.Spec.SandboxClass
43+
if class == "" {
44+
class = atev1alpha1.SandboxClassGvisor
45+
}
46+
47+
var sc *atev1alpha1.SandboxConfig
48+
if name := wp.Spec.SandboxConfigName; name != "" {
49+
sc, err = sandboxConfigLister.Get(name)
50+
if err != nil {
51+
return nil, fmt.Errorf("while getting SandboxConfig %q: %w", name, err)
52+
}
53+
if sc.Spec.SandboxClass != class {
54+
return nil, fmt.Errorf("SandboxConfig %q has class %q but WorkerPool %s/%s is class %q",
55+
name, sc.Spec.SandboxClass, ref.Namespace, ref.Name, class)
56+
}
57+
} else {
58+
sc, err = defaultSandboxConfig(sandboxConfigLister, class)
59+
if err != nil {
60+
return nil, err
61+
}
62+
}
63+
64+
return sandboxAssetsProto(class, sc), nil
65+
}
66+
67+
// defaultSandboxConfig returns the single SandboxConfig marked Default for the
68+
// given class, erroring if there are zero or more than one.
69+
func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev1alpha1.SandboxClass) (*atev1alpha1.SandboxConfig, error) {
70+
all, err := lister.List(labels.Everything())
71+
if err != nil {
72+
return nil, fmt.Errorf("while listing SandboxConfigs: %w", err)
73+
}
74+
var match *atev1alpha1.SandboxConfig
75+
for _, sc := range all {
76+
if sc.Spec.SandboxClass == class && sc.Spec.Default {
77+
if match != nil {
78+
return nil, fmt.Errorf("multiple default SandboxConfigs for class %q (%q and %q)", class, match.Name, sc.Name)
79+
}
80+
match = sc
81+
}
82+
}
83+
if match == nil {
84+
return nil, fmt.Errorf("no default SandboxConfig for class %q; set one with spec.default=true or name one via WorkerPool.spec.sandboxConfigName", class)
85+
}
86+
return match, nil
87+
}
88+
89+
// sandboxAssetsProto converts a resolved SandboxConfig into the proto atelet
90+
// consumes.
91+
func sandboxAssetsProto(class atev1alpha1.SandboxClass, sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets {
92+
out := &ateletpb.SandboxAssets{
93+
SandboxClass: string(class),
94+
Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)),
95+
}
96+
for arch, files := range sc.Spec.Assets {
97+
archAssets := &ateletpb.ArchAssets{Files: make(map[string]*ateletpb.AssetFile, len(files))}
98+
for name, f := range files {
99+
archAssets.Files[name] = &ateletpb.AssetFile{Url: f.URL, Sha256: f.SHA256}
100+
}
101+
out.Assets[arch] = archAssets
102+
}
103+
return out
104+
}

‎cmd/ateapi/internal/controlapi/service.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,19 @@ type Service struct {
3333
var _ ateapipb.ControlServer = (*Service)(nil)
3434

3535
// NewService creates a service.
36-
func NewService(persistence store.Interface, actorTemplateLister listersv1alpha1.ActorTemplateLister, dialer *AteletDialer, kubeClient kubernetes.Interface) *Service {
36+
func NewService(
37+
persistence store.Interface,
38+
actorTemplateLister listersv1alpha1.ActorTemplateLister,
39+
workerPoolLister listersv1alpha1.WorkerPoolLister,
40+
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
41+
dialer *AteletDialer,
42+
kubeClient kubernetes.Interface,
43+
) *Service {
3744
s := &Service{
3845
persistence: persistence,
3946
actorTemplateLister: actorTemplateLister,
4047
dialer: dialer,
41-
actorWorkflow: NewActorWorkflow(persistence, dialer, actorTemplateLister, kubeClient),
48+
actorWorkflow: NewActorWorkflow(persistence, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, kubeClient),
4249
}
4350

4451
return s

‎cmd/ateapi/internal/controlapi/workflow.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,16 +117,27 @@ type ActorWorkflow struct {
117117
store store.Interface
118118
dialer *AteletDialer
119119
actorTemplateLister listersv1alpha1.ActorTemplateLister
120+
workerPoolLister listersv1alpha1.WorkerPoolLister
121+
sandboxConfigLister listersv1alpha1.SandboxConfigLister
120122
kubeClient kubernetes.Interface
121123
secretCache *envSecretCache
122124
}
123125

124126
// NewActorWorkflow creates a new ActorWorkflow.
125-
func NewActorWorkflow(store store.Interface, dialer *AteletDialer, actorTemplateLister listersv1alpha1.ActorTemplateLister, kubeClient kubernetes.Interface) *ActorWorkflow {
127+
func NewActorWorkflow(
128+
store store.Interface,
129+
dialer *AteletDialer,
130+
actorTemplateLister listersv1alpha1.ActorTemplateLister,
131+
workerPoolLister listersv1alpha1.WorkerPoolLister,
132+
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
133+
kubeClient kubernetes.Interface,
134+
) *ActorWorkflow {
126135
return &ActorWorkflow{
127136
store: store,
128137
dialer: dialer,
129138
actorTemplateLister: actorTemplateLister,
139+
workerPoolLister: workerPoolLister,
140+
sandboxConfigLister: sandboxConfigLister,
130141
kubeClient: kubeClient,
131142
secretCache: newEnvSecretCache(envSecretCacheTTL),
132143
}
@@ -151,7 +162,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, id string, boot bool) (
151162
steps := []WorkflowStep[*ResumeInput, *ResumeState]{
152163
&LoadActorForResumeStep{store: w.store, actorTemplateLister: w.actorTemplateLister},
153164
&AssignWorkerStep{store: w.store},
154-
&CallAteletRestoreStep{dialer: w.dialer, kubeClient: w.kubeClient, secretCache: w.secretCache},
165+
&CallAteletRestoreStep{dialer: w.dialer, kubeClient: w.kubeClient, secretCache: w.secretCache, workerPoolLister: w.workerPoolLister, sandboxConfigLister: w.sandboxConfigLister},
155166
&FinalizeRunningStep{store: w.store},
156167
}
157168

‎cmd/ateapi/internal/controlapi/workflow_pause.go‎

Lines changed: 3 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -114,31 +114,14 @@ func (s *CallAteletPauseStep) Execute(ctx context.Context, input *PauseInput, st
114114
}
115115
client := ateletpb.NewAteomHerderClient(ateletConn)
116116

117-
runscCfg := &ateletpb.RunscConfig{}
118-
if state.ActorTemplate.Spec.Runsc.AMD64 != nil {
119-
runscCfg.Amd64 = &ateletpb.RunscPlatformConfig{
120-
Sha256Hash: state.ActorTemplate.Spec.Runsc.AMD64.SHA256Hash,
121-
Url: state.ActorTemplate.Spec.Runsc.AMD64.URL,
122-
}
123-
}
124-
if state.ActorTemplate.Spec.Runsc.ARM64 != nil {
125-
runscCfg.Arm64 = &ateletpb.RunscPlatformConfig{
126-
Sha256Hash: state.ActorTemplate.Spec.Runsc.ARM64.SHA256Hash,
127-
Url: state.ActorTemplate.Spec.Runsc.ARM64.URL,
128-
}
129-
}
130-
if state.ActorTemplate.Spec.Runsc.Authentication.GCP != nil {
131-
authnCfg := &ateletpb.AuthenticationConfig{}
132-
authnCfg.Gcp = &ateletpb.GCPAuthenticationConfig{Use: true}
133-
runscCfg.Authentication = authnCfg
134-
}
135-
117+
// Checkpoint does not carry the sandbox config: atelet uses the version the
118+
// actor is currently running (recorded on-node at Run/Restore) and pins it
119+
// into the snapshot manifest.
136120
req := &ateletpb.CheckpointRequest{
137121
TargetAteomUid: state.Actor.GetAteomPodUid(),
138122
ActorTemplateNamespace: state.Actor.GetActorTemplateNamespace(),
139123
ActorTemplateName: state.Actor.GetActorTemplateName(),
140124
ActorId: state.Actor.GetActorId(),
141-
Runsc: runscCfg,
142125
Spec: &ateletpb.WorkloadSpec{
143126
PauseImage: state.ActorTemplate.Spec.PauseImage,
144127
},

0 commit comments

Comments
 (0)