diff --git a/Makefile b/Makefile index c68dbd3ea..78956d61b 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,8 @@ build-atenet: .PHONY: build-demos build-demos: - $(KO) build --ldflags="$(LDFLAGS)" ./demos/counter + GOFLAGS='"-ldflags=$(LDFLAGS)"' \ + $(KO) build ./demos/counter ./demos/egress .PHONY: test test: diff --git a/cmd/ateapi/internal/controlapi/create_actor.go b/cmd/ateapi/internal/controlapi/create_actor.go index 50eb05473..f35c37ddf 100644 --- a/cmd/ateapi/internal/controlapi/create_actor.go +++ b/cmd/ateapi/internal/controlapi/create_actor.go @@ -67,6 +67,7 @@ func (s *Service) CreateActor(ctx context.Context, req *ateapipb.CreateActorRequ ActorTemplateNamespace: templateNamespace, ActorTemplateName: templateName, WorkerSelector: in.GetWorkerSelector(), + Labels: normalizeLabels(in.GetLabels()), } stored, err := s.persistence.CreateActor(ctx, actor) if err != nil { @@ -121,6 +122,9 @@ func validateCreateActorRequest(req *ateapipb.CreateActorRequest) error { errs = append(errs, validateSelector(val, actorPath.Child("worker_selector"))...) } + errs = append(errs, validateLabels(actor.GetLabels(), actorPath.Child("labels"))...) + errs = append(errs, validateEgressPEPSelector(actor.GetLabels(), actorPath.Child("labels"))...) + if len(errs) > 0 { return status.Error(codes.InvalidArgument, errs.ToAggregate().Error()) } diff --git a/cmd/ateapi/internal/controlapi/create_atespace.go b/cmd/ateapi/internal/controlapi/create_atespace.go index cba8c12ad..1010b1024 100644 --- a/cmd/ateapi/internal/controlapi/create_atespace.go +++ b/cmd/ateapi/internal/controlapi/create_atespace.go @@ -37,6 +37,7 @@ func (s *Service) CreateAtespace(ctx context.Context, req *ateapipb.CreateAtespa Metadata: &ateapipb.ResourceMetadata{ Name: name, }, + Labels: normalizeLabels(req.GetAtespace().GetLabels()), } stored, err := s.persistence.CreateAtespace(ctx, atespace) if err != nil { @@ -71,6 +72,8 @@ func validateCreateAtespaceRequest(req *ateapipb.CreateAtespaceRequest) error { errs = append(errs, resources.ValidateResourceName(val, p)...) } + errs = append(errs, validateLabels(atespace.GetLabels(), atespacePath.Child("labels"))...) + errs = append(errs, validateEgressPEPSelector(atespace.GetLabels(), atespacePath.Child("labels"))...) if len(errs) > 0 { return status.Error(codes.InvalidArgument, errs.ToAggregate().Error()) } diff --git a/cmd/ateapi/internal/controlapi/egress_pep.go b/cmd/ateapi/internal/controlapi/egress_pep.go new file mode 100644 index 000000000..23a1e5d8a --- /dev/null +++ b/cmd/ateapi/internal/controlapi/egress_pep.go @@ -0,0 +1,149 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "fmt" + "net" + + "github.com/agent-substrate/substrate/internal/egress" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "k8s.io/apimachinery/pkg/api/validate/content" + "k8s.io/apimachinery/pkg/util/validation/field" + netutils "k8s.io/utils/net" +) + +// resolveEgressPEPAddress selects the egress PEP address for an actor by +// consumer-driven precedence, following the Istio ambient istio.io/use-waypoint +// model: the actor's own selector wins, then the atespace's, then the global +// default. Each tier supplies the PEP address directly as ":" via +// the ate.dev/use-egress-pep label (the global default comes from the +// --default-egress-pep flag). +// +// ate-api has no dependency on the Gateway API: it does not look up, watch, or +// validate any Gateway resource. The selected address is passed to ateom as +// given. An empty result means no PEP selected and egress capture stays off. A +// malformed address is a configuration error and fails resolution loudly. +func resolveEgressPEPAddress(actor *ateapipb.Actor, atespace *ateapipb.Atespace, defaultAddr string) (string, error) { + tiers := []struct { + source string + address string + }{ + {"actor", actor.GetLabels()[egress.LabelUseEgressPEP]}, + {"atespace", atespace.GetLabels()[egress.LabelUseEgressPEP]}, + {"global", defaultAddr}, + } + + for _, tier := range tiers { + if tier.address == "" { + continue + } + if err := validatePEPAddress(tier.address); err != nil { + return "", fmt.Errorf("%s egress PEP: %w", tier.source, err) + } + return tier.address, nil + } + return "", nil +} + +// validateEgressPEPSelector validates the ate.dev/use-egress-pep selector label +// (if present and non-empty) parses as ":". An absent key means no +// selector; an explicit empty value also means no selector — resolution skips +// empty tiers, and on UpdateActor an empty value deletes the key. +func validateEgressPEPSelector(labels map[string]string, fldPath *field.Path) field.ErrorList { + address := labels[egress.LabelUseEgressPEP] + if address == "" { + return nil + } + if err := validatePEPAddress(address); err != nil { + return field.ErrorList{field.Invalid(fldPath.Key(egress.LabelUseEgressPEP), address, err.Error())} + } + return nil +} + +const ( + // maxLabels bounds the number of selector-label entries on an actor or + // atespace, mirroring maxSelectorMatchLabels for worker selectors. + maxLabels = 10 + // maxLabelValueLength bounds label values. Values are not restricted to the + // Kubernetes label-value charset because the egress PEP selector carries a + // ":" address; the cap allows a maximal DNS name (253) plus a + // colon and port with margin. + maxLabelValueLength = 320 +) + +// validateLabels bounds a request's selector-label map: entry count, Kubernetes +// label-key syntax, and value length. Value contents beyond length are only +// checked for keys with dedicated validators (validateEgressPEPSelector). +func validateLabels(labels map[string]string, fldPath *field.Path) field.ErrorList { + var errs field.ErrorList + if n := len(labels); n > maxLabels { + return field.ErrorList{field.TooMany(fldPath, n, maxLabels)} + } + for k, v := range labels { + for _, msg := range content.IsLabelKey(k) { + errs = append(errs, field.Invalid(fldPath.Key(k), k, msg)) + } + if len(v) > maxLabelValueLength { + errs = append(errs, field.TooLong(fldPath.Key(k), v, maxLabelValueLength)) + } + } + return errs +} + +// ValidateDefaultEgressPEPAddress validates the global-default egress PEP address +// (the --default-egress-pep flag) at boot. Empty is allowed (no global default). +// A malformed non-empty address is a configuration error the caller surfaces so +// a typo does not silently reach ateom and fail every actor resume that falls +// through to the global tier; ate-api logs it and degrades to no global default. +func ValidateDefaultEgressPEPAddress(address string) error { + if address == "" { + return nil + } + return validatePEPAddress(address) +} + +// normalizeLabels returns a copy of labels without empty-valued entries (an +// empty value means "no selector"), or nil when nothing remains. Create paths +// use it so an explicit empty selector is stored the same as an absent one. +func normalizeLabels(labels map[string]string) map[string]string { + var out map[string]string + for k, v := range labels { + if v == "" { + continue + } + if out == nil { + out = map[string]string{} + } + out[k] = v + } + return out +} + +// validatePEPAddress checks that an egress PEP address is a ":" pair +// with a numeric port in range. ate-api does not verify the host resolves or +// that a Gateway actually serves it — the address is used as given. +func validatePEPAddress(address string) error { + host, port, err := net.SplitHostPort(address) + if err != nil || host == "" || port == "" { + return fmt.Errorf("egress PEP address %q must be in the form :", address) + } + // ParsePort rejects sign prefixes ("+80") and out-of-range ports that + // SplitHostPort passes through unchecked. + if _, err := netutils.ParsePort(port, false); err != nil { + return fmt.Errorf("egress PEP address %q has an invalid port %q", address, port) + } + return nil +} diff --git a/cmd/ateapi/internal/controlapi/egress_pep_test.go b/cmd/ateapi/internal/controlapi/egress_pep_test.go new file mode 100644 index 000000000..96d3cfc3c --- /dev/null +++ b/cmd/ateapi/internal/controlapi/egress_pep_test.go @@ -0,0 +1,162 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "fmt" + "strings" + "testing" + + "github.com/agent-substrate/substrate/internal/egress" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +func actorWithPEP(address string) *ateapipb.Actor { + a := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{ + Atespace: "space-a", + Name: "actor-a", + }, + } + if address != "" { + a.Labels = map[string]string{egress.LabelUseEgressPEP: address} + } + return a +} + +func atespaceWithPEP(address string) *ateapipb.Atespace { + s := &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: "space-a"}} + if address != "" { + s.Labels = map[string]string{egress.LabelUseEgressPEP: address} + } + return s +} + +func TestResolveEgressPEPAddressNoSelector(t *testing.T) { + got, err := resolveEgressPEPAddress(actorWithPEP(""), atespaceWithPEP(""), "") + if err != nil { + t.Fatalf("resolveEgressPEPAddress() error = %v", err) + } + if got != "" { + t.Fatalf("resolveEgressPEPAddress() = %q, want empty", got) + } +} + +func TestResolveEgressPEPAddressPrefersActorThenAtespaceThenGlobal(t *testing.T) { + const ( + actorPEP = "ate-egress-actor.agentgateway-system.svc.cluster.local:15008" + spacePEP = "ate-egress-space.agentgateway-system.svc.cluster.local:15008" + globalPEP = "ate-egress.agentgateway-system.svc.cluster.local:15008" + ) + + // Actor selector wins over atespace and global default. + got, err := resolveEgressPEPAddress(actorWithPEP(actorPEP), atespaceWithPEP(spacePEP), globalPEP) + if err != nil { + t.Fatalf("resolveEgressPEPAddress() error = %v", err) + } + if got != actorPEP { + t.Fatalf("resolveEgressPEPAddress() = %q, want %q", got, actorPEP) + } + + // No actor selector: atespace wins over global default. + got, err = resolveEgressPEPAddress(actorWithPEP(""), atespaceWithPEP(spacePEP), globalPEP) + if err != nil { + t.Fatalf("resolveEgressPEPAddress() error = %v", err) + } + if got != spacePEP { + t.Fatalf("resolveEgressPEPAddress() = %q, want %q", got, spacePEP) + } + + // No actor or atespace selector: global default is used. + got, err = resolveEgressPEPAddress(actorWithPEP(""), atespaceWithPEP(""), globalPEP) + if err != nil { + t.Fatalf("resolveEgressPEPAddress() error = %v", err) + } + if got != globalPEP { + t.Fatalf("resolveEgressPEPAddress() = %q, want %q", got, globalPEP) + } +} + +func TestResolveEgressPEPAddressMalformedErrors(t *testing.T) { + if _, err := resolveEgressPEPAddress(actorWithPEP("no-port-here"), atespaceWithPEP(""), ""); err == nil { + t.Fatal("resolveEgressPEPAddress() error = nil, want error for malformed address") + } +} + +func TestValidateDefaultEgressPEPAddress(t *testing.T) { + // Empty is allowed: no global default configured. + if err := ValidateDefaultEgressPEPAddress(""); err != nil { + t.Fatalf("ValidateDefaultEgressPEPAddress(\"\") error = %v, want nil", err) + } + if err := ValidateDefaultEgressPEPAddress("ate-egress.agentgateway-system.svc.cluster.local:15008"); err != nil { + t.Fatalf("ValidateDefaultEgressPEPAddress() valid address error = %v", err) + } + if err := ValidateDefaultEgressPEPAddress("no-port-here"); err == nil { + t.Fatal("ValidateDefaultEgressPEPAddress() malformed address error = nil, want error") + } +} + +func TestValidateEgressPEPSelector(t *testing.T) { + if errs := validateEgressPEPSelector(map[string]string{egress.LabelUseEgressPEP: "pep.example:15008"}, nil); len(errs) != 0 { + t.Fatalf("validateEgressPEPSelector() valid address returned errors: %v", errs) + } + if errs := validateEgressPEPSelector(map[string]string{egress.LabelUseEgressPEP: "missing-port"}, nil); len(errs) == 0 { + t.Fatal("validateEgressPEPSelector() malformed address returned no errors") + } + // A signed port must be rejected (ParsePort, unlike Atoi, refuses "+80"). + if errs := validateEgressPEPSelector(map[string]string{egress.LabelUseEgressPEP: "pep.example:+80"}, nil); len(errs) == 0 { + t.Fatal("validateEgressPEPSelector() signed port returned no errors") + } + if errs := validateEgressPEPSelector(nil, nil); len(errs) != 0 { + t.Fatalf("validateEgressPEPSelector() absent label returned errors: %v", errs) + } + // An explicit empty value means "no selector" (delete on UpdateActor) and + // must be accepted. + if errs := validateEgressPEPSelector(map[string]string{egress.LabelUseEgressPEP: ""}, nil); len(errs) != 0 { + t.Fatalf("validateEgressPEPSelector() empty value returned errors: %v", errs) + } +} + +func TestValidateLabels(t *testing.T) { + if errs := validateLabels(map[string]string{"team": "a", egress.LabelUseEgressPEP: "pep.example:15008"}, nil); len(errs) != 0 { + t.Fatalf("validateLabels() valid labels returned errors: %v", errs) + } + if errs := validateLabels(map[string]string{"not a label key!": "v"}, nil); len(errs) == 0 { + t.Fatal("validateLabels() invalid key returned no errors") + } + if errs := validateLabels(map[string]string{"k": strings.Repeat("v", maxLabelValueLength+1)}, nil); len(errs) == 0 { + t.Fatal("validateLabels() oversized value returned no errors") + } + tooMany := map[string]string{} + for i := 0; i <= maxLabels; i++ { + tooMany[fmt.Sprintf("key-%d", i)] = "v" + } + if errs := validateLabels(tooMany, nil); len(errs) == 0 { + t.Fatal("validateLabels() too many entries returned no errors") + } +} + +func TestNormalizeLabels(t *testing.T) { + got := normalizeLabels(map[string]string{"keep": "v", "drop": ""}) + if len(got) != 1 || got["keep"] != "v" { + t.Fatalf("normalizeLabels() = %v, want only the non-empty entry", got) + } + if got := normalizeLabels(map[string]string{"drop": ""}); got != nil { + t.Fatalf("normalizeLabels() all-empty = %v, want nil", got) + } + if got := normalizeLabels(nil); got != nil { + t.Fatalf("normalizeLabels(nil) = %v, want nil", got) + } +} diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 57c08a9c5..837a94705 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -29,6 +29,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/ateredis" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/egress" "github.com/agent-substrate/substrate/internal/envtestbins" "github.com/agent-substrate/substrate/internal/proto/ateletpb" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -310,7 +311,7 @@ func setupTest(t *testing.T, ns string) *testContext { } dialer := NewAteletDialer(workerInformer.GetIndexer(), ateletInformer.GetIndexer()) - service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient) + service := NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, k8sClient, "") // 5. Start REAL gRPC Server for ATE API grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor)) @@ -1719,6 +1720,75 @@ func TestUpdateActor_Success(t *testing.T) { } } +// TestUpdateActor_MergesLabels verifies UpdateActor merges labels rather than +// replacing them: an absent map leaves labels untouched, a non-empty value sets +// its key, and an empty value deletes it. +func TestUpdateActor_MergesLabels(t *testing.T) { + ns := namespaceForTest("ns-update-actor-labels") + tc := setupTest(t, ns) + defer tc.cleanup() + + createTemplate(t, tc, ns) + + const pep = "pep.example:15008" + actorRef := &ateapipb.ObjectRef{Atespace: testAtespace, Name: "id1"} + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{ + Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.GetAtespace(), Name: actorRef.GetName()}, + ActorTemplateNamespace: ns, + ActorTemplateName: "tmpl1", + Labels: map[string]string{egress.LabelUseEgressPEP: pep, "team": "a"}, + }, + }); err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + + // A labels-unaware update (nil map) must not touch existing labels. + if _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{Actor: actorRef}); err != nil { + t.Fatalf("UpdateActor (nil labels) failed: %v", err) + } + getResp, err := tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + wantLabels := map[string]string{egress.LabelUseEgressPEP: pep, "team": "a"} + if diff := cmp.Diff(wantLabels, getResp.GetLabels()); diff != "" { + t.Errorf("labels after nil-labels update mismatch (-want +got):\n%s", diff) + } + + // A partial update sets its key and leaves the others alone. + if _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{ + Actor: actorRef, + Labels: map[string]string{"team": "b"}, + }); err != nil { + t.Fatalf("UpdateActor (set team) failed: %v", err) + } + getResp, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + wantLabels = map[string]string{egress.LabelUseEgressPEP: pep, "team": "b"} + if diff := cmp.Diff(wantLabels, getResp.GetLabels()); diff != "" { + t.Errorf("labels after partial update mismatch (-want +got):\n%s", diff) + } + + // An empty value deletes its key. + if _, err := tc.client.UpdateActor(context.Background(), &ateapipb.UpdateActorRequest{ + Actor: actorRef, + Labels: map[string]string{egress.LabelUseEgressPEP: ""}, + }); err != nil { + t.Fatalf("UpdateActor (clear egress PEP) failed: %v", err) + } + getResp, err = tc.client.GetActor(context.Background(), &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + t.Fatalf("GetActor failed: %v", err) + } + wantLabels = map[string]string{"team": "b"} + if diff := cmp.Diff(wantLabels, getResp.GetLabels()); diff != "" { + t.Errorf("labels after delete-by-empty mismatch (-want +got):\n%s", diff) + } +} + func TestUpdateActor_NotFound(t *testing.T) { ns := namespaceForTest("ns-update-actor-notfound") tc := setupTest(t, ns) diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 7ec695b00..9aa44ab9b 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -43,13 +43,14 @@ func NewService( sandboxConfigLister listersv1alpha1.SandboxConfigLister, dialer *AteletDialer, kubeClient kubernetes.Interface, + defaultEgressPEP string, ) *Service { s := &Service{ persistence: persistence, actorTemplateLister: actorTemplateLister, workerPoolLister: workerPoolLister, dialer: dialer, - actorWorkflow: NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, kubeClient), + actorWorkflow: NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, kubeClient, defaultEgressPEP), } return s diff --git a/cmd/ateapi/internal/controlapi/syncer.go b/cmd/ateapi/internal/controlapi/syncer.go index 82854f826..3945e32d9 100644 --- a/cmd/ateapi/internal/controlapi/syncer.go +++ b/cmd/ateapi/internal/controlapi/syncer.go @@ -229,6 +229,7 @@ func (s *WorkerPoolSyncer) releaseActorOnDeadWorker(ctx context.Context, namespa actor.AteomPodIp = "" actor.InProgressSnapshot = "" actor.WorkerPoolName = "" + actor.EgressPepAddress = "" if _, err := s.persistence.UpdateActor(ctx, actor, actor.GetMetadata().GetVersion()); err != nil && !errors.Is(err, store.ErrPersistenceRetry) { return err } diff --git a/cmd/ateapi/internal/controlapi/update_actor.go b/cmd/ateapi/internal/controlapi/update_actor.go index db3384121..56630ea65 100644 --- a/cmd/ateapi/internal/controlapi/update_actor.go +++ b/cmd/ateapi/internal/controlapi/update_actor.go @@ -40,6 +40,24 @@ func (s *Service) UpdateActor(ctx context.Context, req *ateapipb.UpdateActorRequ return nil, fmt.Errorf("while getting actor: %w", err) } actor.WorkerSelector = req.GetWorkerSelector() + // Labels are merged, not replaced: an absent/empty map leaves the actor's + // labels untouched (so a labels-unaware caller cannot wipe selectors set by + // someone else), a key with a non-empty value sets it, and a key with an + // empty value deletes it. Proto3 cannot distinguish a nil map from an empty + // one on the wire, so delete-by-empty-value is the explicit clear path. + for k, v := range req.GetLabels() { + if v == "" { + delete(actor.Labels, k) + continue + } + if actor.Labels == nil { + actor.Labels = map[string]string{} + } + actor.Labels[k] = v + } + if len(actor.Labels) == 0 { + actor.Labels = nil + } updated, err := s.persistence.UpdateActor(ctx, actor, actor.GetMetadata().GetVersion()) if err != nil { @@ -66,6 +84,9 @@ func validateUpdateActorRequest(req *ateapipb.UpdateActorRequest) error { errs = append(errs, validateSelector(val, fldPath.Child("worker_selector"))...) } + errs = append(errs, validateLabels(req.GetLabels(), fldPath.Child("labels"))...) + errs = append(errs, validateEgressPEPSelector(req.GetLabels(), fldPath.Child("labels"))...) + if len(errs) > 0 { return status.Error(codes.InvalidArgument, errs.ToAggregate().Error()) } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 496e1bb89..04cdc1de7 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -122,6 +122,7 @@ type ActorWorkflow struct { workerPoolLister listersv1alpha1.WorkerPoolLister sandboxConfigLister listersv1alpha1.SandboxConfigLister kubeClient kubernetes.Interface + defaultEgressPEP string secretCache *envSecretCache } @@ -134,6 +135,7 @@ func NewActorWorkflow( workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, kubeClient kubernetes.Interface, + defaultEgressPEP string, ) *ActorWorkflow { return &ActorWorkflow{ store: store, @@ -143,6 +145,7 @@ func NewActorWorkflow( workerPoolLister: workerPoolLister, sandboxConfigLister: sandboxConfigLister, kubeClient: kubeClient, + defaultEgressPEP: defaultEgressPEP, secretCache: newEnvSecretCache(envSecretCacheTTL), } } @@ -166,7 +169,7 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, atespace, name string, steps := []WorkflowStep[*ResumeInput, *ResumeState]{ &LoadActorForResumeStep{store: w.store, actorTemplateLister: w.actorTemplateLister}, - &AssignWorkerStep{store: w.store, workerCache: w.workerCache}, + &AssignWorkerStep{store: w.store, workerCache: w.workerCache, defaultEgressPEP: w.defaultEgressPEP}, &CallAteletRestoreStep{store: w.store, dialer: w.dialer, kubeClient: w.kubeClient, secretCache: w.secretCache, workerPoolLister: w.workerPoolLister, sandboxConfigLister: w.sandboxConfigLister}, &FinalizeRunningStep{store: w.store}, } diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 4499ac6a5..0ecbfa986 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -222,6 +222,7 @@ func (s *FinalizePausedStep) Execute(ctx context.Context, input *PauseInput, sta latestActor.AteomPodName = "" latestActor.AteomPodIp = "" latestActor.WorkerPoolName = "" + latestActor.EgressPepAddress = "" updatedActor, err := s.store.UpdateActor(ctx, latestActor, latestActor.GetMetadata().GetVersion()) if err != nil { return err diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 3b7c9ab5f..c128406b1 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -106,8 +106,9 @@ func isWorkerEligibleForActor(worker *ateapipb.Worker, templateClass atev1alpha1 } type AssignWorkerStep struct { - store store.Interface - workerCache *workercache.Cache + store store.Interface + workerCache *workercache.Cache + defaultEgressPEP string } func (s *AssignWorkerStep) Name() string { return "AssignWorker" } @@ -116,6 +117,44 @@ func (s *AssignWorkerStep) IsComplete(ctx context.Context, input *ResumeInput, s return state.Actor.GetStatus() == ateapipb.Actor_STATUS_RUNNING, nil } func (s *AssignWorkerStep) Execute(ctx context.Context, input *ResumeInput, state *ResumeState) error { + // Resolve the egress PEP for this actor and record it on the actor so the + // binding is observable (e.g. "which actors use the global PEP?"). Selection + // is consumer-driven (actor > atespace > global default) and re-resolved on + // every resume, so we load the atespace to read its ate.dev/use-egress-pep + // selector. An empty address means no PEP matched and egress capture stays + // off. CallAteletRestoreStep reads this back to tell ateom. Resolved before + // any worker is claimed so a resolution error (e.g. a malformed address) + // fails the resume without stranding a worker assignment. + // A missing atespace record must not strand the actor: actors can predate + // mandatory atespace records or survive the non-atomic AtespaceExists / + // DeleteAtespace race, and before egress selection resume never read the + // atespace at all. GetLabels is nil-safe, so a nil atespace degrades to + // actor-label / global-default resolution. + atespace, err := s.store.GetAtespace(ctx, state.Actor.GetMetadata().GetAtespace()) + if err != nil { + if !errors.Is(err, store.ErrNotFound) { + return fmt.Errorf("while loading atespace for egress PEP resolution: %w", err) + } + slog.WarnContext(ctx, "Atespace record not found during egress PEP resolution; using actor/global tiers only", + "actorId", state.Actor.GetMetadata().GetName(), + "atespace", state.Actor.GetMetadata().GetAtespace()) + atespace = nil + } + egressPEPAddress, err := resolveEgressPEPAddress(state.Actor, atespace, s.defaultEgressPEP) + if err != nil { + return err + } + if egressPEPAddress == "" { + slog.InfoContext(ctx, "Resolved no egress PEP for actor; egress capture disabled", + "actorId", state.Actor.GetMetadata().GetName(), + "atespace", state.Actor.GetMetadata().GetAtespace()) + } else { + slog.InfoContext(ctx, "Resolved egress PEP for actor", + "actorId", state.Actor.GetMetadata().GetName(), + "atespace", state.Actor.GetMetadata().GetAtespace(), + "pepAddress", egressPEPAddress) + } + workers, err := s.workerCache.Workers() if err != nil { return fmt.Errorf("while listing workers: %w", err) @@ -190,6 +229,7 @@ func (s *AssignWorkerStep) Execute(ctx context.Context, input *ResumeInput, stat state.Actor.AteomPodIp = assignedWorker.GetIp() state.Actor.AteomPodUid = assignedWorker.GetWorkerPodUid() state.Actor.WorkerPoolName = assignedWorker.GetWorkerPool() + state.Actor.EgressPepAddress = egressPEPAddress updatedActor, err := s.store.UpdateActor(ctx, state.Actor, state.Actor.GetMetadata().GetVersion()) if err != nil { @@ -265,6 +305,9 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, if err != nil { return err } + // AssignWorkerStep resolved and persisted the egress PEP for this resume; read + // it back to tell ateom where to tunnel captured egress (empty = no redirect). + egressPEPAddress := state.Actor.GetEgressPepAddress() if data := state.Actor.GetLatestSnapshotInfo().GetData(); data != nil { slog.InfoContext(ctx, "Actor has snapshot; Restoring from snapshot") @@ -275,6 +318,7 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, ActorName: state.Actor.GetMetadata().GetName(), ActorTemplateNamespace: state.Actor.GetActorTemplateNamespace(), ActorTemplateName: state.Actor.GetActorTemplateName(), + EgressPepAddress: egressPEPAddress, Spec: workloadSpec, } switch d := data.(type) { @@ -311,6 +355,7 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, ActorName: state.Actor.GetMetadata().GetName(), ActorTemplateNamespace: state.Actor.GetActorTemplateNamespace(), ActorTemplateName: state.Actor.GetActorTemplateName(), + EgressPepAddress: egressPEPAddress, Spec: workloadSpec, Type: ateletpb.CheckpointType_CHECKPOINT_TYPE_EXTERNAL, Config: &ateletpb.RestoreRequest_ExternalConfig{ @@ -339,6 +384,7 @@ func (s *CallAteletRestoreStep) Execute(ctx context.Context, input *ResumeInput, ActorName: state.Actor.GetMetadata().GetName(), ActorTemplateNamespace: state.Actor.GetActorTemplateNamespace(), ActorTemplateName: state.Actor.GetActorTemplateName(), + EgressPepAddress: egressPEPAddress, SandboxAssets: sandboxAssets, Spec: workloadSpec, } diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 6a4cc6cc4..058c9f23c 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -214,6 +214,7 @@ func (s *FinalizeSuspendedStep) Execute(ctx context.Context, input *SuspendInput latestActor.AteomPodName = "" latestActor.AteomPodIp = "" latestActor.WorkerPoolName = "" + latestActor.EgressPepAddress = "" updatedActor, err := s.store.UpdateActor(ctx, latestActor, latestActor.GetMetadata().GetVersion()) if err != nil { return err diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index dd481bc50..b13baf643 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -70,6 +70,8 @@ var ( sessionIDCAPoolFile = pflag.String("session-id-ca-pool", "", "The file that contains the CA pool for signing session JWTs") workerpoolCACerts = pflag.String("workerpool-ca-certs", "", "The file that contains the CA for verifying workerpool client certificates.") + defaultEgressPEP = pflag.String("default-egress-pep", "", "The global-default egress PEP address, as :. Used for actors and atespaces that set no ate.dev/use-egress-pep selector. Empty disables the global default (capture off unless a per-actor/atespace selector matches).") + showVersion = pflag.Bool("version", false, "Print version and exit.") authMode = pflag.String("auth-mode", "mtls", "Auth mode for incoming gRPC: mtls|jwt. 'mtls' (default) relies on transport-level mTLS for client identity. 'jwt' additionally requires a Kubernetes ServiceAccount Bearer token on every RPC. Substrate will drop support for JWT auth mode once the Pod Certificates feature is enabled by default in the minimum supported Kubernetes version.") clientJWTCAFile = pflag.String("client-jwt-ca-cert", ateapiauth.DefaultServiceAccountCAFile, "CA cert file used to verify TLS when fetching the OIDC discovery document and JWKS for JWT authentication. Defaults to the in-cluster service account CA.") @@ -100,6 +102,19 @@ func main() { defer serverboot.ShutdownProvider("MeterProvider", mp.Shutdown) loadFlagsFromEnv() + + // A malformed --default-egress-pep degrades to "no global default" rather than + // crash-looping the deployment: the flag only affects actors that fall through + // to the global tier, and per-actor/atespace selectors keep working. We clear + // it so the bad value can't reach ateom and fail every fall-through resume at + // connect time; the warning is the operator's signal to fix the flag. Cleared + // before logFlagValues so the "Final flag values" line reports the effective + // (empty) value, not the ignored one. + if err := controlapi.ValidateDefaultEgressPEPAddress(*defaultEgressPEP); err != nil { + slog.WarnContext(ctx, "Ignoring invalid --default-egress-pep; global-default egress PEP disabled", "err", err) + *defaultEgressPEP = "" + } + logFlagValues(ctx) authModeParsed, err := ateapiauth.ParseMode(*authMode) @@ -151,7 +166,7 @@ func main() { ateFactory.WaitForCacheSync(stopCh) dialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer()) - sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset) + sm := controlapi.NewService(redisPersistence, workerCache, actorTemplateLister, workerPoolLister, sandboxConfigLister, dialer, clientset, *defaultEgressPEP) jwtIssuerDiscoveryClient := buildK8sServiceAccountIssuerDiscoveryClient(ctx, *clientJWTCAFile, *clientJWTIssuer) if authModeParsed == ateapiauth.ModeJWT && jwtIssuerDiscoveryClient == nil { @@ -216,6 +231,7 @@ func loadFlagsFromEnv() { {redisUseIAMAuth, "ATE_API_REDIS_USE_IAM_AUTH"}, {redisTLSServerName, "ATE_API_REDIS_TLS_SERVER_NAME"}, {redisClientCert, "ATE_API_REDIS_CLIENT_CERT"}, + {defaultEgressPEP, "ATE_API_DEFAULT_EGRESS_PEP"}, } for _, o := range overrides { if *o.flag == "@env" { @@ -239,6 +255,7 @@ func logFlagValues(ctx context.Context) { slog.String("session-id-ca-pool", *sessionIDCAPoolFile), slog.String("workerpool-ca-certs", *workerpoolCACerts), slog.String("auth-mode", *authMode), + slog.String("default-egress-pep", *defaultEgressPEP), ) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index faa94f572..6a586d2af 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -44,7 +44,8 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool) *appsv1ac.Deployment WithValueFrom(corev1ac.EnvVarSource(). WithFieldRef(corev1ac.ObjectFieldSelector(). WithFieldPath("metadata.uid"))), - ). + ) + containerAC. WithVolumeMounts(corev1ac.VolumeMount(). WithName("run-ateom"). WithMountPath(ateompath.BasePath)) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 9a5eaefaa..df6826ab3 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -259,6 +259,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (*atele ActorName: actorName, ActorTemplateNamespace: req.GetActorTemplateNamespace(), ActorTemplateName: req.GetActorTemplateName(), + EgressPepAddress: req.GetEgressPepAddress(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, Spec: buildAteomWorkloadSpec(req.GetSpec()), @@ -554,6 +555,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) ActorName: actorName, ActorTemplateNamespace: req.GetActorTemplateNamespace(), ActorTemplateName: req.GetActorTemplateName(), + EgressPepAddress: req.GetEgressPepAddress(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, Spec: buildAteomWorkloadSpec(req.GetSpec()), diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 5e856e7bd..5fdfa27e7 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -30,8 +30,10 @@ import ( "cloud.google.com/go/compute/metadata" "github.com/agent-substrate/substrate/internal/actorlog" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/ateomegress" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/contextlogging" + "github.com/agent-substrate/substrate/internal/egress" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/serverboot" @@ -164,6 +166,7 @@ type AteomService struct { interiorNetNS netns.NsHandle actorLogger *actorlog.ActorLogger + egressCapture *egress.Capture } var _ ateompb.AteomServer = (*AteomService)(nil) @@ -188,7 +191,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // * Correct runsc version is downloaded and placed on disk. // * All OCI bundles are set up, including for "pause" container. - if err := s.setupActorNetwork(ctx); err != nil { + if err := s.setupActorNetwork(ctx, ateomegress.ActorIdentityFromRun(req), req.GetEgressPepAddress()); err != nil { return nil, fmt.Errorf("while setting up actor network: %w", err) } defer func() { @@ -362,7 +365,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // * All OCI bundles are set up, including for "pause" container. // * Checkpoint downloaded and placed on disk - if err := s.setupActorNetwork(ctx); err != nil { + if err := s.setupActorNetwork(ctx, ateomegress.ActorIdentityFromRestore(req), req.GetEgressPepAddress()); err != nil { return nil, fmt.Errorf("while setting up actor network: %w", err) } defer func() { @@ -438,7 +441,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return &ateompb.RestoreWorkloadResponse{}, nil } -func (s *AteomService) setupActorNetwork(ctx context.Context) (retErr error) { +func (s *AteomService) setupActorNetwork(ctx context.Context, identity egress.ActorIdentity, egressPEPAddress string) (retErr error) { // Build a fresh point-to-point network between the worker pod netns and the // gVisor interior netns. The worker side keeps the pod's real eth0, creates // ateom0 as the gateway, and moves only the veth peer into the actor netns. @@ -506,7 +509,13 @@ func (s *AteomService) setupActorNetwork(ctx context.Context) (retErr error) { if err := enableIPv4Forwarding(); err != nil { return err } - if err := installActorNftablesRules(podIP); err != nil { + egressCapture, err := ateomegress.StartCaptureIfEnabled(ctx, identity, egressPEPAddress) + if err != nil { + return err + } + s.egressCapture = egressCapture + + if err := installActorNftablesRules(podIP, s.egressCapture != nil); err != nil { return err } @@ -590,6 +599,12 @@ func (s *AteomService) cleanupActorNetwork(ctx context.Context) error { if err := removeActorNftablesRules(); err != nil { return err } + if s.egressCapture != nil { + if err := s.egressCapture.Close(); err != nil { + slog.WarnContext(ctx, "Failed to close actor egress capture", "err", err) + } + s.egressCapture = nil + } var cleanupErr error if link, err := netlink.LinkByName(hostVethName); err == nil { @@ -671,7 +686,7 @@ func enableIPv4Forwarding() error { return nil } -func installActorNftablesRules(podIP net.IP) error { +func installActorNftablesRules(podIP net.IP, egressCapture bool) error { // Install a dedicated nftables table for the active actor. Keeping all // rules in an ateom-owned table makes cleanup simple and avoids mutating // Kubernetes or CNI-managed chains directly. @@ -709,6 +724,13 @@ func installActorNftablesRules(podIP net.IP) error { Hooknum: nftables.ChainHookPrerouting, Priority: nftables.ChainPriorityNATDest, }) + if egressCapture { + ateomegress.AddCaptureRedirectRules(c, table, prerouting, actorVethIP) + } + // TODO: Support optional DNS capture for hostname recovery for non-SNI, + // non-HTTP, or DNS-policy egress. The current HTTP/HTTPS path derives + // authority from TLS SNI or HTTP Host, so redirecting UDP/TCP 53 would + // add potential DNS proxy/cache/TTL/search-domain failures // TODO: Support inbound UDP DNAT for actors that expose UDP protocols such // as QUIC. // TODO: Replace the hard-coded HTTP port with the actor's configured diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index c1998da20..6a5c85626 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -36,6 +36,7 @@ import ( "github.com/agent-substrate/substrate/internal/actorlog" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/egress" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/serverboot" "github.com/agent-substrate/substrate/internal/version" @@ -205,6 +206,10 @@ type AteomService struct { // with ateom-gvisor). actorLogger *actorlog.ActorLogger + // egressCapture owns the per-activation capture listener when egress capture is + // enabled for this worker pod. + egressCapture *egress.Capture + // running maps actor name -> the live micro-VM, kept so CheckpointWorkload can // pause+snapshot+teardown the same sandbox (and RestoreWorkload can track the // CH it relaunched). diff --git a/cmd/ateom-microvm/net.go b/cmd/ateom-microvm/net.go index cc047d4f2..1aa3c7a4a 100644 --- a/cmd/ateom-microvm/net.go +++ b/cmd/ateom-microvm/net.go @@ -29,9 +29,6 @@ package main // CONSTANT, a restored guest's frozen network config stays valid on any pod — // no in-guest reconfiguration needed. // -// (Copied with light adaptation from cmd/ateom-gvisor; expected to be -// de-duplicated into a shared package later.) - import ( "context" "errors" @@ -48,6 +45,8 @@ import ( "github.com/vishvananda/netns" "golang.org/x/sys/unix" + "github.com/agent-substrate/substrate/internal/ateomegress" + "github.com/agent-substrate/substrate/internal/egress" "github.com/agent-substrate/substrate/internal/serverboot" ) @@ -120,7 +119,7 @@ func mustParseIP(s string) net.IP { // pod netns and the kata interior netns (see the package comment). Idempotent // via cleanup-before-setup; also sweeps stale kata taps out of the interior // netns so the sandbox always builds on a clean slate. -func (s *AteomService) setupActorNetwork(ctx context.Context) (retErr error) { +func (s *AteomService) setupActorNetwork(ctx context.Context, identity egress.ActorIdentity, egressPEPAddress string) (retErr error) { s.cleanupActorNetworkOrExit(ctx, "Failed to clean up stale actor network before setup") defer func() { if retErr != nil { @@ -190,7 +189,12 @@ func (s *AteomService) setupActorNetwork(ctx context.Context) (retErr error) { if err := enableIPv4Forwarding(); err != nil { return err } - if err := installActorNftablesRules(podIP); err != nil { + egressCapture, err := ateomegress.StartCaptureIfEnabled(ctx, identity, egressPEPAddress) + if err != nil { + return err + } + s.egressCapture = egressCapture + if err := installActorNftablesRules(podIP, s.egressCapture != nil); err != nil { return err } @@ -249,6 +253,12 @@ func (s *AteomService) cleanupActorNetwork(ctx context.Context) error { cleanupErr = errors.Join(cleanupErr, fmt.Errorf("while removing actor nftables rules: %w", err)) slog.WarnContext(ctx, "Failed to remove actor nftables rules; continuing actor netns cleanup", slog.Any("err", err)) } + if s.egressCapture != nil { + if err := s.egressCapture.Close(); err != nil { + slog.WarnContext(ctx, "Failed to close actor egress capture", slog.Any("err", err)) + } + s.egressCapture = nil + } if link, err := netlink.LinkByName(hostVethName); err == nil { if err := netlink.LinkDel(link); err != nil { @@ -333,7 +343,7 @@ func enableIPv4Forwarding() error { return nil } -func installActorNftablesRules(podIP net.IP) error { +func installActorNftablesRules(podIP net.IP, egressCapture bool) error { // Dedicated ateom-owned IPv4 table (cheap cleanup, no CNI chain mutation): // * postrouting: masquerade actor egress (169.254.17.2) behind the pod IP. // * prerouting: DNAT pod-IP:80/tcp to the actor veth IP. @@ -357,6 +367,9 @@ func installActorNftablesRules(podIP net.IP) error { Hooknum: nftables.ChainHookPrerouting, Priority: nftables.ChainPriorityNATDest, }) + if egressCapture { + ateomegress.AddCaptureRedirectRules(c, table, prerouting, actorVethIP) + } preroutingExprs := append(ipDestinationEqual(podIP.String()), tcpDestinationPortEqual(80)...) preroutingExprs = append(preroutingExprs, &expr.Immediate{ diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 54407693e..425dda050 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -28,6 +28,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/ateomegress" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" @@ -112,7 +113,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. - if err := s.setupActorNetwork(ctx); err != nil { + if err := s.setupActorNetwork(ctx, ateomegress.ActorIdentityFromRestore(req), req.GetEgressPepAddress()); err != nil { return nil, fmt.Errorf("while setting up actor network: %w", err) } defer func() { diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 2038e550a..e2b56b46f 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -29,6 +29,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/third_party/kata/agentpb" + "github.com/agent-substrate/substrate/internal/ateomegress" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" @@ -218,7 +219,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // Networking (host side): per-activation veth into the interior netns. The // tap + TC mirror is built below (after the VM exists) so its FDs are fresh. - if err := s.setupActorNetwork(ctx); err != nil { + if err := s.setupActorNetwork(ctx, ateomegress.ActorIdentityFromRun(req), req.GetEgressPepAddress()); err != nil { return nil, fmt.Errorf("while setting up actor network: %w", err) } defer func() { diff --git a/cmd/kubectl-ate/internal/cmd/create_actor.go b/cmd/kubectl-ate/internal/cmd/create_actor.go index 6a8c750cd..9e967c28d 100644 --- a/cmd/kubectl-ate/internal/cmd/create_actor.go +++ b/cmd/kubectl-ate/internal/cmd/create_actor.go @@ -20,12 +20,14 @@ import ( "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/egress" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" ) var templateFlag string var atespaceFlag string +var createActorEgressPEPFlag string var createActorCmd = &cobra.Command{ Use: "actor ", @@ -45,6 +47,11 @@ var createActorCmd = &cobra.Command{ return fmt.Errorf("malformed --template: %s (expected /)", templateFlag) } + var labels map[string]string + if createActorEgressPEPFlag != "" { + labels = map[string]string{egress.LabelUseEgressPEP: createActorEgressPEPFlag} + } + resp, err := apiClient.CreateActor(ctx, &ateapipb.CreateActorRequest{ Actor: &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{ @@ -53,6 +60,7 @@ var createActorCmd = &cobra.Command{ }, ActorTemplateNamespace: parts[0], ActorTemplateName: parts[1], + Labels: labels, }, }) if err != nil { @@ -68,5 +76,6 @@ func init() { _ = createActorCmd.MarkFlagRequired("template") createActorCmd.Flags().StringVarP(&atespaceFlag, "atespace", "a", "", "Atespace to create the actor in (required)") _ = createActorCmd.MarkFlagRequired("atespace") + createActorCmd.Flags().StringVar(&createActorEgressPEPFlag, "egress-pep", "", "Egress PEP address for this actor, as :. Sets the ate.dev/use-egress-pep selector (highest precedence: actor > atespace > global default).") createCmd.AddCommand(createActorCmd) } diff --git a/cmd/kubectl-ate/internal/cmd/create_atespace.go b/cmd/kubectl-ate/internal/cmd/create_atespace.go index 518e31fae..cdd2e695d 100644 --- a/cmd/kubectl-ate/internal/cmd/create_atespace.go +++ b/cmd/kubectl-ate/internal/cmd/create_atespace.go @@ -19,10 +19,13 @@ import ( "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/egress" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/spf13/cobra" ) +var createAtespaceEgressPEPFlag string + var createAtespaceCmd = &cobra.Command{ Use: "atespace [name]", Short: "Create an atespace", @@ -35,11 +38,17 @@ var createAtespaceCmd = &cobra.Command{ } defer apiClient.Close() + var labels map[string]string + if createAtespaceEgressPEPFlag != "" { + labels = map[string]string{egress.LabelUseEgressPEP: createAtespaceEgressPEPFlag} + } + resp, err := apiClient.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{ Atespace: &ateapipb.Atespace{ Metadata: &ateapipb.ResourceMetadata{ Name: args[0], }, + Labels: labels, }, }) if err != nil { @@ -51,5 +60,6 @@ var createAtespaceCmd = &cobra.Command{ } func init() { + createAtespaceCmd.Flags().StringVar(&createAtespaceEgressPEPFlag, "egress-pep", "", "Egress PEP address for all actors in this atespace, as :. Sets the ate.dev/use-egress-pep selector (precedence: actor > atespace > global default).") createCmd.AddCommand(createAtespaceCmd) } diff --git a/cmd/kubectl-ate/internal/cmd/update.go b/cmd/kubectl-ate/internal/cmd/update.go new file mode 100644 index 000000000..998548b37 --- /dev/null +++ b/cmd/kubectl-ate/internal/cmd/update.go @@ -0,0 +1,28 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/spf13/cobra" +) + +var updateCmd = &cobra.Command{ + Use: "update", + Short: "Update a resource", +} + +func init() { + rootCmd.AddCommand(updateCmd) +} diff --git a/cmd/kubectl-ate/internal/cmd/update_actor.go b/cmd/kubectl-ate/internal/cmd/update_actor.go new file mode 100644 index 000000000..84350acb1 --- /dev/null +++ b/cmd/kubectl-ate/internal/cmd/update_actor.go @@ -0,0 +1,81 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/agent-substrate/substrate/cmd/kubectl-ate/internal/printer" + "github.com/agent-substrate/substrate/internal/ateclient" + "github.com/agent-substrate/substrate/internal/egress" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "github.com/spf13/cobra" +) + +var updateActorAtespaceFlag string +var updateActorEgressPEPFlag string + +var updateActorCmd = &cobra.Command{ + Use: "actor ", + Short: "Update an actor", + Long: "Update mutable fields on an actor. Changes take effect on the next resume. " + + "Use --egress-pep to set the ate.dev/use-egress-pep selector to a : " + + "PEP address; pass an empty value to clear it. Other mutable fields (e.g. the " + + "worker selector) are preserved.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + apiClient, err := ateclient.NewClient(ctx, kubeconfig, k8sContext, endpoint, traceEnabled) + if err != nil { + return fmt.Errorf("failed to connect to ate-api-server: %w", err) + } + defer apiClient.Close() + + actorRef := &ateapipb.ObjectRef{Atespace: updateActorAtespaceFlag, Name: args[0]} + + // UpdateActor merges labels (empty value deletes the key), so only the + // changed label is sent. The worker selector is still replaced wholesale, + // so read the current actor first to echo it back unchanged. + // TODO: UpdateActorRequest carries no version token, so a concurrent + // update landing between this read and the write below is silently lost. + current, err := apiClient.GetActor(ctx, &ateapipb.GetActorRequest{Actor: actorRef}) + if err != nil { + return fmt.Errorf("failed to get actor: %w", err) + } + + var labels map[string]string + if cmd.Flags().Changed("egress-pep") { + labels = map[string]string{egress.LabelUseEgressPEP: updateActorEgressPEPFlag} + } + + resp, err := apiClient.UpdateActor(ctx, &ateapipb.UpdateActorRequest{ + Actor: actorRef, + WorkerSelector: current.GetWorkerSelector(), + Labels: labels, + }) + if err != nil { + return fmt.Errorf("failed to update actor: %w", err) + } + + return printer.PrintActor(resp.GetActor(), outputFmt) + }, +} + +func init() { + updateActorCmd.Flags().StringVarP(&updateActorAtespaceFlag, "atespace", "a", "", "Atespace the actor lives in (required)") + _ = updateActorCmd.MarkFlagRequired("atespace") + updateActorCmd.Flags().StringVar(&updateActorEgressPEPFlag, "egress-pep", "", "Egress PEP address for this actor, as :. Sets the ate.dev/use-egress-pep selector; empty clears it. Takes effect on the next resume.") + updateCmd.AddCommand(updateActorCmd) +} diff --git a/demos/egress/.gitignore b/demos/egress/.gitignore new file mode 100644 index 000000000..659600455 --- /dev/null +++ b/demos/egress/.gitignore @@ -0,0 +1,2 @@ +*.test +/egress diff --git a/demos/egress/README.md b/demos/egress/README.md new file mode 100644 index 000000000..8a9b18e34 --- /dev/null +++ b/demos/egress/README.md @@ -0,0 +1,68 @@ +# Egress Demo + +This demo deploys a small HTTP actor that opens outbound HTTP or HTTPS requests +from inside the actor sandbox. It is intended for validating actor egress +capture and agentgateway routing. + +## Run + +Install the CLI as a kubectl plugin if needed: + +```bash +go install ./cmd/kubectl-ate +export PATH="$(go env GOPATH)/bin:${PATH}" +``` + +Deploy the ATE system with egress capture enabled, then deploy the demo: + +```bash +./hack/install-ate.sh --egress --deploy-ate-system +./hack/install-ate.sh --deploy-demo-egress +``` + +For kind, use the kind wrapper for both commands: + +```bash +./hack/install-ate-kind.sh --egress --deploy-ate-system +./hack/install-ate-kind.sh --deploy-demo-egress +``` + +Create an actor and port-forward the router: + +```bash +kubectl ate create actor my-egress-1 --template ate-demo-egress/egress +kubectl port-forward -n ate-system svc/atenet-router 8000:80 +``` + +Call the actor: + +```bash +curl -X POST \ + -H "Host: my-egress-1.actors.resources.substrate.ate.dev" \ + http://localhost:8000 +``` + +The default target is `https://httpbin.org/get`. To call another httpbin path: + +```bash +curl -X POST \ + -H "Host: my-egress-1.actors.resources.substrate.ate.dev" \ + --data-urlencode "url=https://httpbin.org/headers" \ + "http://localhost:8000" +``` + +The demo agentgateway setup only routes `httpbin.org:443`. + +## Uninstall + +```bash +kubectl ate suspend actor my-egress-1 +kubectl ate delete actor my-egress-1 +./hack/install-ate.sh --delete-demo-egress +``` + +For kind: + +```bash +./hack/install-ate-kind.sh --delete-demo-egress +``` diff --git a/demos/egress/egress.yaml.tmpl b/demos/egress/egress.yaml.tmpl new file mode 100644 index 000000000..e6f9b080f --- /dev/null +++ b/demos/egress/egress.yaml.tmpl @@ -0,0 +1,56 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-demo-egress + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: egress + namespace: ate-demo-egress + labels: + workload: egress +spec: + replicas: 2 + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: egress + namespace: ate-demo-egress +spec: + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + containers: + - name: egress + image: ko://github.com/agent-substrate/substrate/demos/egress + command: ["/ko-app/egress"] + readyz: + httpGet: + path: /readyz + port: 80 + workerSelector: + matchLabels: + workload: egress + snapshotsConfig: + onPause: Full + onCommit: Full + location: gs://${BUCKET_NAME}/ate-demo-egress/ diff --git a/demos/egress/main.go b/demos/egress/main.go new file mode 100644 index 000000000..742ca49fe --- /dev/null +++ b/demos/egress/main.go @@ -0,0 +1,111 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Command egress is a small HTTP workload for exercising captured actor egress. +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "time" +) + +const defaultEgressURL = "https://httpbin.org/get" + +func main() { + ctx := context.Background() + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) + + mux := http.NewServeMux() + mux.HandleFunc("/", handleEgress) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok\n")) + }) + + slog.InfoContext(ctx, "Starting egress demo server on port 80") + if err := http.ListenAndServe(":80", mux); err != nil { + slog.ErrorContext(ctx, "Error starting server", slog.Any("err", err)) + os.Exit(1) + } +} + +func handleEgress(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + targetURL, err := egressTargetURL(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) + if err != nil { + http.Error(w, fmt.Sprintf("invalid egress target %q: %v", targetURL, err), http.StatusBadRequest) + return + } + + start := time.Now() + resp, err := http.DefaultClient.Do(req) + if err != nil { + slog.ErrorContext(ctx, "Egress request failed", slog.String("target", targetURL), slog.Any("err", err)) + http.Error(w, fmt.Sprintf("egress request to %s failed: %v\n", targetURL, err), http.StatusBadGateway) + return + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) + if err != nil { + slog.ErrorContext(ctx, "Failed reading egress response", slog.String("target", targetURL), slog.Any("err", err)) + http.Error(w, fmt.Sprintf("reading egress response from %s failed: %v\n", targetURL, err), http.StatusBadGateway) + return + } + + slog.InfoContext(ctx, "Egress request completed", + slog.String("target", targetURL), + slog.Int("upstream_status", resp.StatusCode), + slog.Duration("duration", time.Since(start))) + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "egress target: %s\n", targetURL) + fmt.Fprintf(w, "upstream status: %s\n", resp.Status) + fmt.Fprintf(w, "body bytes read: %d\n", len(body)) + fmt.Fprintf(w, "body:\n%s\n", body) +} + +func egressTargetURL(r *http.Request) (string, error) { + targetURL := r.URL.Query().Get("url") + if targetURL == "" { + targetURL = defaultEgressURL + } + + parsed, err := url.Parse(targetURL) + if err != nil { + return "", fmt.Errorf("invalid egress target %q: %w", targetURL, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", fmt.Errorf("invalid egress target %q: scheme must be http or https", targetURL) + } + if parsed.Host == "" { + return "", fmt.Errorf("invalid egress target %q: host is required", targetURL) + } + return targetURL, nil +} diff --git a/demos/egress/main_test.go b/demos/egress/main_test.go new file mode 100644 index 000000000..5267ec46f --- /dev/null +++ b/demos/egress/main_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http/httptest" + "testing" +) + +func TestEgressTargetURL(t *testing.T) { + for _, tc := range []struct { + name string + path string + want string + wantErr bool + }{ + { + name: "default", + path: "/", + want: defaultEgressURL, + }, + { + name: "custom url", + path: "/?url=https%3A%2F%2Fhttpbin.org%2Fheaders", + want: "https://httpbin.org/headers", + }, + { + name: "reject missing host", + path: "/?url=https%3A%2F%2F", + wantErr: true, + }, + { + name: "reject unsupported scheme", + path: "/?url=ftp%3A%2F%2Fexample.com%2Ffile", + wantErr: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest("POST", tc.path, nil) + got, err := egressTargetURL(req) + if tc.wantErr { + if err == nil { + t.Fatal("egressTargetURL() returned nil error, want error") + } + return + } + if err != nil { + t.Fatalf("egressTargetURL() returned error: %v", err) + } + if got != tc.want { + t.Fatalf("egressTargetURL() = %q, want %q", got, tc.want) + } + }) + } +} diff --git a/docs/egress-capture.md b/docs/egress-capture.md new file mode 100644 index 000000000..1cad80472 --- /dev/null +++ b/docs/egress-capture.md @@ -0,0 +1,637 @@ +# Egress Capture + +This documents the demo install path and the ateom capture setup using +the dedicated egress demo. The demo is a small HTTP actor that resumes through +the router and opens an outbound HTTPS request to `https://httpbin.org/get` +by default. + +## Architecture + +Egress capture has no global on/off switch. PEP selection is **consumer-driven**, +following the Istio ambient waypoint model: an actor (or its atespace, or a +global default) names the egress PEP it wants via the `ate.dev/use-egress-pep` +selector, exactly as Istio's `istio.io/use-waypoint` names a waypoint. The +difference from Istio is that the selector's value is the PEP **address** +(`:`) directly — ate-api has **no Gateway API dependency at all**: it +does not watch, look up, or validate any Gateway resource. On actor resume it +reads the actor's selector (falling back to the atespace's, then the +`--default-egress-pep` flag) and sends that address to ateom as given. An empty +address means no redirect, so capture is enabled per actor only when a selector +resolves to an address. The reusable capture core lives in `internal/egress`: it +owns capture listeners, authority derivation, CONNECT tunnel transports, and byte +proxying. The runtime-specific `ateom` egress proxy setup supplies the +original-destination lookup and packet-capture rules. + +The current gVisor and MicroVM implementations start a local capture listener +and install actor-network redirects for TCP egress. From the actor's point of +view it still opens a normal TCP connection to the original destination. Future +hypervisor implementations should reuse `internal/egress` for the local +listener, authority derivation, tunnel transport, and byte proxying. Each +runtime still provides its own egress proxy setup for redirecting actor traffic +and recovering the original destination. + +The redirected connection lands on `ateom`, which records the original +destination and derives a stable CONNECT authority from the first bytes of the +actor connection: + +| Actor traffic | Authority source | Example CONNECT authority | +| --- | --- | --- | +| HTTPS / any TCP port | TLS ClientHello SNI + original destination port | `httpbin.org:443` | +| Plaintext HTTP / any TCP port | HTTP `Host` header, defaulting to original destination port when the header has no port | `example.com:80` | +| Other TCP | Original destination address | `203.0.113.10:2222` | + +The shared capture core then opens a plaintext HTTP/2 CONNECT stream to the PEP +address ate-api resolved for the actor. That address is whatever the selector +supplied — for the demo, the agentgateway service DNS name +`ate-egress.agentgateway-system.svc.cluster.local:15008`. ate-api does not +resolve or health-check it; a wrong address simply fails the tunnel at connect +time. Agentgateway maps the CONNECT authority to its configured TCP listener and +routes the tunnel to a Kubernetes Service backed by an EndpointSlice. + +The demo setup configures only `httpbin.org:443` for egress. Any other CONNECT +authority, including plaintext HTTP destinations or fallback original IP:port +authorities, needs its own matching agentgateway Service, EndpointSlice, +listener, and route. For HTTPS, TLS is still end-to-end between the actor and +the external service; agentgateway only routes the encrypted bytes after +CONNECT succeeds. + +### Selecting a PEP for an actor + +Selection lives on the consumer via the `ate.dev/use-egress-pep` label, whose +value is the PEP address `:`. ate-api reads it from three tiers and +uses the highest-precedence one that is set: + +| Selector | Scope | Precedence | +| --- | --- | --- | +| `--default-egress-pep` flag on ate-api | Global (any actor) | lowest | +| `ate.dev/use-egress-pep` on the **Atespace** | All actors in the atespace | medium | +| `ate.dev/use-egress-pep` on the **Actor** | One actor | highest | + +On resume, ate-api walks actor → atespace → global default in order and uses the +first tier whose value is set (`resolveEgressPEPAddress` in +`cmd/ateapi/internal/controlapi/egress_pep.go`). The value is passed straight +through to ateom; ate-api never contacts the Gateway API. + +#### Fall-through + +| Situation | Result | +| --- | --- | +| A tier's selector is unset | Skipped; ate-api uses the next tier | +| No tier is set | Empty PEP address → no redirect, capture off | +| An actor/atespace selector is not a valid `:` | Configuration error; rejected at `CreateActor` / `UpdateActor` / `CreateAtespace`, and the resume fails loudly as defense-in-depth | +| The global default (`--default-egress-pep`) is not a valid `:` | ate-api logs a warning at startup and degrades to no global default (the value is cleared, never sent to ateom); actor/atespace selectors are unaffected | + +Each tier supplies exactly one address, so selection is unambiguous — there is no +tie-breaking. + +#### Trust model + +The control surface is the ate-api API. Who can point an actor at a PEP is +governed entirely by RBAC on `CreateActor` / `UpdateActor` / `CreateAtespace` +and on the `--default-egress-pep` flag (set at install). Because the selector +carries a raw address and ate-api enforces no allowlist, anyone who can set an +actor/atespace selector can direct that actor's egress tunnel to any reachable +`:`. Restrict those RPCs to the platform team. (This trades the old +Gateway-label allowlist for zero Gateway API dependency — an explicit choice.) + +#### When the binding is (re)computed + +The PEP binding is a **snapshot taken at resume**, recorded on the actor as +`egress_pep_address`. Selector changes have no effect on RUNNING actors. + +``` + create + │ + ▼ + SUSPENDED ──────────────────────────────────┐ + │ resume / boot │ + ▼ │ + RESUMING ── resolve PEP now: │ + │ AssignWorkerStep reads the │ + │ actor/atespace/global selector │ + │ and writes its address to │ + │ actor.egress_pep_address │ + ▼ │ + RUNNING ── uses the PEP captured at │ + │ resume for its whole lifetime; │ + │ selector changes are IGNORED │ + │ suspend / pause │ + ▼ │ + SUSPENDED / PAUSED ── egress_pep_address ─────┘ + cleared; re-resolved + on the next resume +``` + +The full resume / egress / suspend sequence, component by component: + +```mermaid +sequenceDiagram + autonumber + + participant CLI as kubectl ate + participant API as ate-api + participant ST as Redis/Valkey + participant LET as atelet + participant OM as ateom + participant CAP as capture listener
:15001 + participant PEP as agentgateway PEP
:15008 + participant EXT as httpbin.org:443 + + rect rgb(235, 243, 255) + Note over CLI,ST: Resume + CLI->>API: resume actor my-egress-1 -a demo + API->>ST: Load suspended actor + atespace + API->>API: Resolve PEP address
actor > atespace > global default + API->>ST: Claim worker + API->>ST: Set RESUMING and EgressPepAddress + end + + rect rgb(235, 255, 238) + Note over API,OM: Restore workload + API->>LET: Run/Restore with EgressPepAddress + LET->>OM: RunWorkload/RestoreWorkload with EgressPepAddress + OM->>OM: setupActorNetwork
veth plus netns + OM->>CAP: Start capture listener + OM->>OM: nftables redirects actor TCP to :15001 + OM-->>LET: ok + LET-->>API: ok + API->>ST: Set RUNNING + end + + rect rgb(255, 248, 225) + Note over OM,EXT: Actor egress connection + OM->>CAP: Actor conn (nftables redirect) to httpbin.org:443 + CAP->>CAP: Get original destination
classify SNI, Host, or original dst + CAP->>PEP: HTTP/2 CONNECT httpbin.org:443
with actor metadata + PEP->>EXT: Route via TCPRoute + Note over CAP,EXT: TLS remains end-to-end
PEP routes encrypted bytes only + CAP-->>OM: Proxy byte stream + end + + rect rgb(255, 235, 238) + Note over CLI,ST: Suspend (pause clears EgressPepAddress the same way) + CLI->>API: suspend actor + API->>ST: Set SUSPENDING + API->>LET: Checkpoint + LET->>OM: CheckpointWorkload + OM->>CAP: Close listener and streams + OM-->>LET: snapshot files + LET-->>API: ok (snapshot uploaded) + API->>ST: Release worker
clear EgressPepAddress + Note over API,ST: Next resume re-resolves the selector + end +``` + +To move a running actor to a different PEP: change its `ate.dev/use-egress-pep` +selector **and** cycle the actor (see "Point an actor at a different PEP"). + +## Prerequisites + +- A working Kubernetes cluster and kubeconfig. +- `kubectl`, `helm`, `jq`, and `curl`. +- `kubectl ate` available from this repo, for example: + +```bash +go install ./cmd/kubectl-ate +export PATH="$(go env GOPATH)/bin:${PATH}" +``` + +## Install with capture enabled + +For a normal cluster: + +```bash +./hack/install-ate.sh --egress --deploy-ate-system +``` + +For kind: + +```bash +./hack/install-ate-kind.sh --egress --deploy-ate-system +``` + +This deploys agentgateway with a static `httpbin.org:443` egress route, sets +ate-api's +`--default-egress-pep=ate-egress.agentgateway-system.svc.cluster.local:15008` +(the global-default selector address), and deploys the ATE system. Actors and +atespaces can override the default with an `ate.dev/use-egress-pep` selector. + +A malformed `--default-egress-pep` does not crash-loop ate-api; it logs a +warning at startup and degrades to no global default (see the "Fall-through" +table). Grep ate-api startup logs for `Ignoring invalid --default-egress-pep` +to catch a typo. + +The install script resolves `httpbin.org` during install and creates the +`httpbin-egress` Service and EndpointSlice for those IPs. For the default HTTPS +demo request, `ateom` derives the CONNECT authority from TLS SNI and the +original destination port. + +Verify the static agentgateway resources: + +```bash +kubectl get gateway -n agentgateway-system ate-egress +kubectl get tcproute -n agentgateway-system httpbin-egress +kubectl get agentgatewaypolicy -n agentgateway-system ate-egress-connect +kubectl get service -n agentgateway-system httpbin-egress +kubectl get endpointslice -n agentgateway-system httpbin-egress +``` + +Expected resources include: + +```text +gateway.gateway.networking.k8s.io/ate-egress +tcproute.gateway.networking.k8s.io/httpbin-egress +agentgatewaypolicy.agentgateway.dev/ate-egress-connect +service/httpbin-egress +endpointslice.discovery.k8s.io/httpbin-egress +``` + +ate-api does not read these resources; the Gateway is the agentgateway PEP the +selector address points at. No `ate.dev/egress-pep` marker is needed. + +## Deploy and call the egress actor + +Deploy the egress demo: + +```bash +./hack/install-ate.sh --deploy-demo-egress +``` + +For kind, keep using the kind wrapper so the demo uses the same local image +registry and snapshot bucket settings: + +```bash +./hack/install-ate-kind.sh --deploy-demo-egress +``` + +Create an actor: + +```bash +kubectl ate create atespace demo +kubectl ate create actor my-egress-1 --template ate-demo-egress/egress -a demo +``` + +Forward the router locally: + +```bash +kubectl port-forward -n ate-system svc/atenet-router 8000:80 +``` + +From another terminal, send an external request through the router. The actor +will make an outbound HTTPS request to `https://httpbin.org/get` by default: + +```bash +curl -i -X POST \ + -H "Host: my-egress-1.demo.actors.resources.substrate.ate.dev" \ + http://localhost:8000 +``` + +Expected response: + +```text +HTTP/1.1 200 OK +egress target: https://httpbin.org/get +upstream status: 200 OK +body bytes read: ... +``` + +The response must name `https://httpbin.org/get`. That proves the actor opened a +TCP connection to `httpbin.org:443` from inside the sandbox. + +To test a different `httpbin.org` path, pass it as the `url` query parameter: + +```bash +curl -i -X POST --get \ + -H "Host: my-egress-1.demo.actors.resources.substrate.ate.dev" \ + --data-urlencode "url=https://httpbin.org/headers" \ + "http://localhost:8000" +``` + +Do not use this query parameter for a different host or port unless you also +update the agentgateway route. `ateom` will derive the new CONNECT authority +from TLS SNI, the HTTP `Host` header, or the original destination, but the demo +agentgateway config only routes `httpbin.org:443`. + +## Verify capture was installed + +Find the worker pod hosting the actor: + +```bash +actor_json=$(kubectl ate get actor my-egress-1 -a demo -o json) +ateom_ns=$(jq -r '.actors[0].ateomPodNamespace' <<<"${actor_json}") +ateom_pod=$(jq -r '.actors[0].ateomPodName' <<<"${actor_json}") + +echo "${ateom_ns}/${ateom_pod}" +``` + +Check the ateom logs: + +```bash +kubectl logs -n "${ateom_ns}" "${ateom_pod}" -c ateom | grep "Started actor egress capture listener" +``` + +Expected output includes one log line for the local capture listener: + +```text +Started actor egress capture listener ... "port":15001 ... "pepAddress":"ate-egress.agentgateway-system.svc.cluster.local:15008" +``` + +After the egress request, the logs should also show the captured stream: + +```bash +kubectl logs -n "${ateom_ns}" "${ateom_pod}" -c ateom | grep "Proxying captured actor egress" +``` + +Expected output includes: + +```text +Proxying captured actor egress ... "originalDestination":"...:443" ... "connectAuthority":"httpbin.org:443" +``` + +## Check which PEP an actor uses + +ate-api records the PEP it resolved for the actor on its most recent resume, so +you can read the binding directly instead of inferring it from selectors. + +From the actor status (`null` means no PEP matched — the field is omitted from +JSON when empty — so capture is off): + +```bash +kubectl ate get actor my-egress-1 -a demo -o json | jq -r '.actors[0].egressPepAddress' +``` + +Expected value for the demo: + +```text +ate-egress.agentgateway-system.svc.cluster.local:15008 +``` + +ate-api also logs the resolution on each resume. This is the easiest way to +answer "which actors fall through to a global PEP" — grep the PEP address across +ate-api logs: + +```bash +kubectl logs -n ate-system deploy/ate-api-server-deployment | grep "Resolved egress PEP for actor" +``` + +Expected output includes: + +```text +Resolved egress PEP for actor ... "actorId":"my-egress-1" "atespace":"demo" "pepAddress":"ate-egress.agentgateway-system.svc.cluster.local:15008" +``` + +Actors that matched no PEP log `Resolved no egress PEP for actor; egress capture +disabled` instead, and their `egressPepAddress` status field is absent. + +## Point an actor at a different PEP + +Suppose you want `my-egress-1` to egress through a second Gateway, +`ate-egress-alt`, instead of the shared `ate-egress` PEP. Because an actor's PEP +is a snapshot taken at resume (see "When the binding is (re)computed"), you both +set the actor's selector and cycle the actor. You do not touch the actor's +current Gateway — the actor's `ate.dev/use-egress-pep` selector out-ranks the +atespace and global-default tiers regardless. + +1. Create the alternate Gateway. No labels are needed on it — ate-api never reads + Gateways; selection happens entirely on the actor: + +```bash +kubectl apply -f - <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: ate-egress-alt + namespace: agentgateway-system +spec: + gatewayClassName: agentgateway + listeners: + - name: connect + port: 15008 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same + - name: https + port: 443 + protocol: TCP + allowedRoutes: + kinds: + - group: gateway.networking.k8s.io + kind: TCPRoute + namespaces: + from: Same +EOF +``` + + The `connect` listener on `15008` is the address the selector will point at + (`ate-egress-alt.agentgateway-system.svc.cluster.local:15008`). + +2. Give the alternate Gateway the CONNECT policy and a route to the backend. + The Gateway alone is not enough: without these the tunnel opens but agentgateway + has nothing routing `httpbin.org:443`, and egress fails with a 502 / connection + reset. The `httpbin-egress` Service and EndpointSlice created by the installer + are backend resources and can be reused as-is; you only need a CONNECT policy + and a TCPRoute parent for the new Gateway: + + ```bash +kubectl apply -f - <<'EOF' +apiVersion: agentgateway.dev/v1alpha1 +kind: AgentgatewayPolicy +metadata: + name: ate-egress-alt-connect + namespace: agentgateway-system +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: ate-egress-alt + frontend: + connect: + mode: Tunnel +EOF + ``` + + Attach the existing `httpbin-egress` TCPRoute to the alternate Gateway by + adding it as a second parent (this leaves `ate-egress` routing intact): + + ```bash + kubectl patch tcproute -n agentgateway-system httpbin-egress --type=json \ + -p '[{"op":"add","path":"/spec/parentRefs/-","value":{"group":"gateway.networking.k8s.io","kind":"Gateway","name":"ate-egress-alt","sectionName":"https"}}]' + ``` + + Confirm both the policy and route attached before continuing: + + ```bash + kubectl get agentgatewaypolicy -n agentgateway-system ate-egress-alt-connect + kubectl get tcproute -n agentgateway-system httpbin-egress \ + -o jsonpath='{range .status.parents[*]}{.parentRef.name}{" Accepted="}{.conditions[?(@.type=="Accepted")].status}{"\n"}{end}' + ``` + +3. Point the actor at the alternate Gateway and cycle it so ate-api re-resolves + the PEP. A running actor keeps its old PEP until it is suspended (or paused) + and resumed: + + ```bash + kubectl ate update actor my-egress-1 -a demo \ + --egress-pep ate-egress-alt.agentgateway-system.svc.cluster.local:15008 + kubectl ate suspend actor my-egress-1 -a demo + kubectl ate resume actor my-egress-1 -a demo + ``` + + `update actor --egress-pep` sets the actor's `ate.dev/use-egress-pep` label to + the given address. Equivalently, set it once at creation with `kubectl ate + create actor ... --egress-pep :`, for example: + + ```bash + kubectl ate create actor my-egress-1 --template ate-demo-egress/egress -a demo \ + --egress-pep ate-egress-alt.agentgateway-system.svc.cluster.local:15008 + ``` + + To scope a whole atespace, set the selector when you create the atespace: + + ```bash + kubectl ate create atespace demo \ + --egress-pep ate-egress.agentgateway-system.svc.cluster.local:15008 + ``` + + The atespace-tier selector can only be set at atespace creation — there is no + `kubectl ate update atespace`. To change the atespace default afterward, + override it per actor with `update actor --egress-pep` (highest precedence); + new actors can pass `--egress-pep` at creation so they never inherit the old + default. Recreating the atespace is only an option while it is empty: + `delete atespace` refuses a non-empty atespace, so once actors exist the + per-actor override is the practical path. + +4. Confirm the actor now points at the alternate PEP: + + ```bash + kubectl ate get actor my-egress-1 -a demo -o json | jq -r '.actors[0].egressPepAddress' + ``` + + Expected value: + + ```text + ate-egress-alt.agentgateway-system.svc.cluster.local:15008 + ``` + +5. Drive traffic again and verify it goes through the new PEP: + + ```bash + curl -i -X POST \ + -H "Host: my-egress-1.demo.actors.resources.substrate.ate.dev" \ + http://localhost:8000 + ``` + + The authoritative signal is the ateom capture log, which names the PEP + configured for the actor's current activation (see "Verify capture was + installed" for how to find the worker pod): + + ```bash + kubectl logs -n "${ateom_ns}" "${ateom_pod}" -c ateom | grep "Started actor egress capture listener" + # ... "pepAddress":"ate-egress-alt.agentgateway-system.svc.cluster.local:15008" + ``` + + On the agentgateway side, match any request the alt Gateway handled rather + than a specific protocol — the CONNECT frontend logs `protocol=http`, and a + `protocol=tcp` line only appears once bytes tunnel end to end (a 5xx from the + upstream can prevent it): + + ```bash + kubectl logs -n agentgateway-system \ + -l gateway.networking.k8s.io/gateway-name=ate-egress-alt \ + --all-containers --tail=200 | grep "gateway=agentgateway-system/ate-egress-alt" + ``` + +To revert, clear the actor's selector so it falls back to the global default +(`ate-egress`), then cycle it; afterward you can remove the alternate Gateway's +route parent and delete the Gateway and its CONNECT policy: + +```bash +kubectl ate update actor my-egress-1 -a demo --egress-pep "" +kubectl ate suspend actor my-egress-1 -a demo +kubectl ate resume actor my-egress-1 -a demo + +kubectl patch tcproute -n agentgateway-system httpbin-egress --type=json \ + -p '[{"op":"remove","path":"/spec/parentRefs/1"}]' +kubectl delete gateway -n agentgateway-system ate-egress-alt +kubectl delete agentgatewaypolicy -n agentgateway-system ate-egress-alt-connect +``` + +## Check agentgateway logs + +The `ate-egress` Gateway creates an agentgateway dataplane pod in the +`agentgateway-system` namespace. Check dataplane logs with: + +```bash +kubectl logs -n agentgateway-system \ + -l gateway.networking.k8s.io/gateway-name=ate-egress \ + --all-containers --tail=200 +``` + +After a successful egress request, dataplane logs should include a TCP route +entry similar to: + +```text +request gateway=agentgateway-system/ate-egress listener=https route=agentgateway-system/httpbin-egress ... protocol=tcp +``` + +If the Gateway, TCPRoute, or policy is not being programmed, check the +agentgateway controller logs: + +```bash +kubectl logs -n agentgateway-system deploy/agentgateway --tail=200 +``` + +## Clean up + +```bash +kubectl ate suspend actor my-egress-1 -a demo +kubectl ate delete actor my-egress-1 -a demo +./hack/install-ate.sh --delete-demo-egress +``` + +## Troubleshooting + +If redeploying fails with `The ActorTemplate "egress" is invalid: spec: +Invalid value: Spec is immutable`, recreate the demo resources: + +```bash +./hack/install-ate-kind.sh --delete-demo-egress +./hack/install-ate-kind.sh --deploy-demo-egress +``` + +If capture listener logs are missing after setting an actor/atespace selector on +an already-running ATE system, no ate-api restart is needed — the selector is +read from the actor and atespace at resume. The usual cause is that the actor was +already running: the PEP binding is a snapshot taken at resume, so cycle the +actor (suspend, then resume) to re-resolve. Selector changes never affect a +RUNNING actor. Confirm which address resolved from ate-api logs +(`Resolved egress PEP for actor`) or the actor's `egressPepAddress` field; if the +address is wrong, the tunnel fails at connect time in ateom. + +Changing the global default (`--default-egress-pep`) does require an ate-api +restart, since it is a process flag / ConfigMap value read at boot: + +```bash +kubectl rollout restart deployment/ate-api-server-deployment -n ate-system +kubectl rollout status deployment/ate-api-server-deployment -n ate-system +``` + +Capture is decided per resume from the PEP address ate-api sends to ateom, so +worker pods do not need to carry any egress config or be restarted. An actor +already running before its selector was set picks up capture on its next resume. + +Worker images must include egress support: an older ateom silently ignores the +PEP address, so `egressPepAddress` on the actor can report a binding the +sandbox does not enforce. If capture logs are missing despite a resolved PEP, +check the WorkerPool's ateom image version. + +If the egress request fails after changing the `url` host or port, remember that +this demo only configures agentgateway for `httpbin.org:443`. Add matching static +agentgateway backend resources for the CONNECT authority that `ateom` will send: + +- HTTPS: SNI plus the original destination port, for example `example.com:443`. +- Plaintext HTTP: `Host` header authority, defaulting to the original + destination port when the header has no port, for example `example.com:80`. +- Other TCP: captured original destination IP and port, for example + `203.0.113.10:2222`. diff --git a/go.mod b/go.mod index 0bc9e101e..c7fd2c8b9 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 + golang.org/x/net v0.55.0 golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.20.0 golang.org/x/sys v0.45.0 @@ -176,7 +177,6 @@ require ( go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.52.0 // indirect - golang.org/x/net v0.55.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.15.0 // indirect diff --git a/hack/install-ate.sh b/hack/install-ate.sh index e7e910a24..48e6dcfc2 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -40,6 +40,7 @@ ATE_DEMOS=() # Include demos. source "${ROOT}"/hack/install-demo-counter.sh +source "${ROOT}"/hack/install-demo-egress.sh source "${ROOT}"/hack/install-demo-sandbox.sh source "${ROOT}"/hack/install-demo-claude-code-multiplex.sh source "${ROOT}"/hack/install-demo-agent-secret.sh @@ -49,6 +50,8 @@ source "${ROOT}"/hack/install-demo-multi-template.sh COLOR_CYAN='\033[1;36m' COLOR_RESET='\033[0m' +ATE_EGRESS_CAPTURE="${ATE_EGRESS_CAPTURE:-false}" + function log_step() { local step_name="$1" echo -e "${COLOR_CYAN}[step]: ${step_name}${COLOR_RESET}" @@ -69,7 +72,9 @@ function usage() { echo "" echo " --deploy-atelet Deploy atelet only" echo " --deploy-ate-apiserver Deploy ate-api-server only" + echo " --deploy-ate-controller Deploy ate-controller only" echo " --deploy-atenet Deploy atenet only" + echo " --egress Enable actor egress capture via labeled agentgateway PEP Gateways" echo "" echo "To create individual resources used by ate-system (Note: These are" echo "called automatically by --deploy-ate-system):" @@ -103,6 +108,12 @@ run_kubectl() { "$@" } +run_helm() { + helm \ + ${KUBECTL_CONTEXT:+--kube-context=${KUBECTL_CONTEXT}} \ + "$@" +} + run_kubectl_ate() { go run ./cmd/kubectl-ate \ ${KUBECTL_CONTEXT:+--context=${KUBECTL_CONTEXT}} \ @@ -167,6 +178,171 @@ render_ate_system_manifests() { fi } +resolve_ipv4_addresses() { + local host="$1" + if command -v python3 >/dev/null 2>&1; then + python3 - "${host}" <<'PY' +import socket +import sys + +host = sys.argv[1] +addresses = sorted({ + result[4][0] + for result in socket.getaddrinfo(host, None, socket.AF_INET, socket.SOCK_STREAM) +}) +print(" ".join(addresses)) +PY + return + fi + if command -v python >/dev/null 2>&1; then + python - "${host}" <<'PY' +import socket +import sys + +host = sys.argv[1] +addresses = sorted(set( + result[4][0] + for result in socket.getaddrinfo(host, None, socket.AF_INET, socket.SOCK_STREAM) +)) +print(" ".join(addresses)) +PY + return + fi + if command -v dig >/dev/null 2>&1; then + dig +short A "${host}" | tr '\n' ' ' + return + fi + if command -v nslookup >/dev/null 2>&1; then + nslookup "${host}" | awk '/^Address: / { print $2 }' | tr '\n' ' ' + return + fi + echo "unable to resolve ${host}: python3, python, dig, or nslookup is required" >&2 + return 1 +} + +ensure_gateway_api() { + log_step "ensure_gateway_api" + if run_kubectl get crd \ + gatewayclasses.gateway.networking.k8s.io \ + gateways.gateway.networking.k8s.io \ + tcproutes.gateway.networking.k8s.io >/dev/null 2>&1; then + return + fi + + run_kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.5.0/experimental-install.yaml +} + +deploy_agentgateway() { + log_step "deploy_agentgateway" + local httpbin_ips="${HTTPBIN_EGRESS_IPS:-}" + if [[ -z "${httpbin_ips}" ]]; then + httpbin_ips="$(resolve_ipv4_addresses httpbin.org)" + fi + if [[ -z "${httpbin_ips}" ]]; then + echo "failed to resolve httpbin.org IPv4 addresses" >&2 + exit 1 + fi + + local httpbin_endpoints="" + local ip="" + for ip in ${httpbin_ips}; do + httpbin_endpoints+=" - addresses:\n - ${ip}\n" + done + + ensure_gateway_api + run_helm upgrade -i --create-namespace \ + --namespace agentgateway-system \ + --version v1.3.1 agentgateway-crds oci://cr.agentgateway.dev/charts/agentgateway-crds + run_helm upgrade -i -n agentgateway-system agentgateway oci://cr.agentgateway.dev/charts/agentgateway \ + --version v1.3.1 + run_kubectl delete deployment -n agentgateway-system ate-egress --ignore-not-found >/dev/null 2>&1 || true + run_kubectl delete service -n agentgateway-system ate-egress --ignore-not-found >/dev/null 2>&1 || true + run_kubectl delete service -n agentgateway-system httpbin-egress --ignore-not-found >/dev/null 2>&1 || true + run_kubectl delete endpointslice -n agentgateway-system httpbin-egress --ignore-not-found >/dev/null 2>&1 || true + run_kubectl delete gateway -n agentgateway-system ate-egress --ignore-not-found >/dev/null 2>&1 || true + run_kubectl delete agentgatewaypolicy -n agentgateway-system ate-egress-connect --ignore-not-found >/dev/null 2>&1 || true + printf "%b" "$(cat < atespace > global). Left + # empty otherwise, which keeps egress capture off. ate-api uses this address + # directly; it has no Gateway API dependency. + local default_egress_pep="" + if [[ "${ATE_EGRESS_CAPTURE}" == "true" ]]; then + default_egress_pep="ate-egress.agentgateway-system.svc.cluster.local:15008" + fi + echo "DEFAULT_EGRESS_PEP: ${default_egress_pep}" + run_kubectl create configmap -n ate-system ate-api-server-envvars \ --from-literal=ATE_API_REDIS_ADDRESS="${redis_address}" \ --from-literal=ATE_API_REDIS_USE_IAM_AUTH="${use_iam_auth}" \ --from-literal=ATE_API_REDIS_TLS_SERVER_NAME="${tls_server_name}" \ --from-literal=ATE_API_REDIS_CLIENT_CERT="${client_cert}" \ --from-literal=ATE_API_K8SJWT_ISSUER="${jwt_issuer}" \ + --from-literal=ATE_API_DEFAULT_EGRESS_PEP="${default_egress_pep}" \ --dry-run=client -o yaml \ | run_kubectl apply -f - } @@ -269,6 +457,10 @@ deploy_ate_system() { log_step "deploy_ate_system" ensure_crds + if [[ "${ATE_EGRESS_CAPTURE}" == "true" ]]; then + deploy_agentgateway + fi + # Enforce per-class SandboxConfig asset requirements (applied before any # SandboxConfig so the defaults below are validated too). run_kubectl apply -f manifests/ate-install/sandboxconfig-validation.yaml @@ -330,6 +522,13 @@ deploy_ate_apiserver() { log_step "deploy_ate_apiserver" ensure_crds + # ate-api is the component that watches PEP Gateways, and it checks for the + # Gateway API once at startup — deploy agentgateway (which installs the + # Gateway API CRDs) before the apiserver rolls. + if [[ "${ATE_EGRESS_CAPTURE}" == "true" ]]; then + deploy_agentgateway + fi + # Ensure namespace exists run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s @@ -360,6 +559,12 @@ deploy_atelet() { run_kubectl rollout status daemonset/atelet -n ate-system --timeout=120s } +deploy_ate_controller() { + log_step "deploy_ate_controller" + run_ko apply -f manifests/ate-install/ate-controller.yaml + run_kubectl rollout status deployment/ate-controller -n ate-system --timeout=120s +} + deploy_atenet() { log_step "deploy_atenet" ensure_crds @@ -376,8 +581,8 @@ deploy_atenet() { # get_actor_status echoes the actor's status enum (e.g. STATUS_SUSPENDED). get_actor_status() { - local actor_id="$1" - local atespace="$2" + local atespace="$1" + local actor_id="$2" local json if ! json=$(run_kubectl_ate get actor "${actor_id}" -a "${atespace}" -o json 2>/dev/null); then @@ -389,14 +594,14 @@ get_actor_status() { # prepare_actor_for_delete suspends (or resumes then suspends) until DeleteActor # is allowed. Actors must be STATUS_SUSPENDED before deletion. prepare_actor_for_delete() { - local actor_id="$1" - local atespace="$2" + local atespace="$1" + local actor_id="$2" local timeout_secs="${3:-120}" local deadline=$((SECONDS + timeout_secs)) local status while ((SECONDS < deadline)); do - if ! status=$(get_actor_status "${actor_id}" "${atespace}"); then + if ! status=$(get_actor_status "${atespace}" "${actor_id}"); then return 0 fi @@ -451,7 +656,7 @@ delete_demo_actors() { return 0 fi - local ns tmpl atespace actor_id + local ns tmpl actor_ref atespace actor_id while (($# > 0)); do ns="$1" tmpl="$2" @@ -459,13 +664,14 @@ delete_demo_actors() { log_step "Deleting actors for ${ns}/${tmpl}" while IFS=$'\t' read -r atespace actor_id; do - [[ -z "${actor_id}" ]] && continue - log_step " preparing actor ${atespace}/${actor_id} for delete" - prepare_actor_for_delete "${actor_id}" "${atespace}" + [[ -z "${atespace}" || -z "${actor_id}" ]] && continue + actor_ref="${atespace}/${actor_id}" + log_step " preparing actor ${actor_ref} for delete" + prepare_actor_for_delete "${atespace}" "${actor_id}" run_kubectl_ate delete actor "${actor_id}" -a "${atespace}" done < <( jq -r --arg ns "${ns}" --arg tmpl "${tmpl}" \ - '.actors[]? | select(.actorTemplateNamespace == $ns and .actorTemplateName == $tmpl) | "\(.atespace)\t\(.actorId)"' \ + '.actors[]? | select(.actorTemplateNamespace == $ns and .actorTemplateName == $tmpl) | [.atespace, .actorId] | @tsv' \ <<<"${actors_json}" ) done @@ -519,6 +725,9 @@ for arg in "$@"; do usage exit 0 ;; + --egress) + ATE_EGRESS_CAPTURE="true" + ;; esac done @@ -577,9 +786,11 @@ while [[ "$#" -gt 0 ]]; do --deploy-atelet) deploy_atelet ;; --deploy-ate-apiserver) deploy_ate_apiserver ;; + --deploy-ate-controller) deploy_ate_controller ;; --deploy-atenet) deploy_atenet ;; --delete-atenet) delete_atenet ;; + --egress) ;; --deploy-benchmarks) deploy_benchmarks ;; --delete-benchmarks) delete_benchmarks ;; diff --git a/hack/install-demo-egress.sh b/hack/install-demo-egress.sh new file mode 100644 index 000000000..09cc2d216 --- /dev/null +++ b/hack/install-demo-egress.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is sourced as part of install-ate.sh. Do not run directly. + +ATE_DEMOS+=(demo-egress) # register demo-egress + +demo-egress_cmdline() { + case "${1}" in + --deploy-demo-egress) demo-egress_deploy ;; + --delete-demo-egress) demo-egress_delete ;; + *) + return 1 + ;; + esac + return 0 +} + +demo-egress_deploy() { + log_step "demo-egress_deploy" + ensure_crds + run_kubectl create namespace ate-demo-egress --dry-run=client -o yaml \ + | run_kubectl apply -f - + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" demos/egress/egress.yaml.tmpl \ + | run_ko apply -f - + + log_step "Waiting for egress demo to be ready..." + run_kubectl rollout status deployment/egress-deployment -n ate-demo-egress --timeout=300s + log_step "Waiting for egress ActorTemplate to be ready..." + run_kubectl wait --for=condition=Ready actortemplate/egress -n ate-demo-egress --timeout=300s +} + +demo-egress_delete() { + log_step "demo-egress_delete" + delete_demo_actors ate-demo-egress egress + sed "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" demos/egress/egress.yaml.tmpl \ + | run_kubectl delete --ignore-not-found -f - +} diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 2e29f6ba6..ed77eb305 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -31,12 +31,24 @@ # OUT asset dir (default: $PWD/bin/microvm-assets/$ARCH, gitignored). # ATE_INSTALL_KIND "true" for the kind path (stage assets to rustfs + install-ate-kind.sh); # default false uploads assets to GCS + uses install-ate.sh. +# +# Flags: +# --egress Enable actor egress capture via agentgateway (forwarded to +# install-ate.sh / install-ate-kind.sh). set -o errexit -o nounset -o pipefail ROOT="$(git rev-parse --show-toplevel)" cd "${ROOT}" +# Parse flags before sourcing the environment so they are available to all steps. +EGRESS_FLAG="" +for arg in "$@"; do + case "${arg}" in + --egress) EGRESS_FLAG="--egress" ;; + esac +done + # Source the environment (cluster, registry, bucket) like the other hack scripts; # hack/run-microvm-demo-kind.sh sets NO_DEV_ENV to skip this and use kind defaults. if [[ -r .ate-dev-env.sh ]] && [[ -z "${NO_DEV_ENV:-}" ]]; then @@ -52,6 +64,7 @@ ATE_API_AUTH_MODE="${ATE_API_AUTH_MODE:-mtls}" while [[ $# -gt 0 ]]; do case "$1" in + --egress) EGRESS_FLAG="--egress" ;; --auth-mode=*) ATE_API_AUTH_MODE="${1#*=}" ;; --auth-mode) if [[ $# -lt 2 ]]; then @@ -131,10 +144,10 @@ fi log "Deploying the ate control plane (--deploy-ate-system)..." if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" ${EGRESS_FLAG} else # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system --auth-mode="${ATE_API_AUTH_MODE}" ${EGRESS_FLAG} fi # --- 4. apply the demo ------------------------------------------------------ diff --git a/internal/ateomegress/doc.go b/internal/ateomegress/doc.go new file mode 100644 index 000000000..6a2b35755 --- /dev/null +++ b/internal/ateomegress/doc.go @@ -0,0 +1,17 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateomegress contains egress-capture helpers shared by ateom +// implementations. +package ateomegress diff --git a/internal/ateomegress/egress_proxy_linux.go b/internal/ateomegress/egress_proxy_linux.go new file mode 100644 index 000000000..48e5e7c3d --- /dev/null +++ b/internal/ateomegress/egress_proxy_linux.go @@ -0,0 +1,156 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomegress + +import ( + "context" + "encoding/binary" + "fmt" + "net" + "syscall" + "unsafe" + + "github.com/agent-substrate/substrate/internal/egress" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" + "golang.org/x/sys/unix" +) + +func StartCaptureIfEnabled(ctx context.Context, identity egress.ActorIdentity, egressPEPAddress string) (*egress.Capture, error) { + cfg, ok := egress.ConfigForPEPAddress(egressPEPAddress, egress.DefaultCaptureListeners) + if !ok { + return nil, nil + } + capture, err := egress.Start(ctx, identity, cfg, originalDestination) + if err != nil { + return nil, fmt.Errorf("while starting actor egress capture: %w", err) + } + return capture, nil +} + +func AddCaptureRedirectRules(c *nftables.Conn, table *nftables.Table, prerouting *nftables.Chain, sourceIP string) { + c.AddRule(&nftables.Rule{ + Table: table, + Chain: prerouting, + Exprs: tcpRedirectExprs(sourceIP, egress.DefaultCapturePort), + }) +} + +func ActorIdentityFromRun(req *ateompb.RunWorkloadRequest) egress.ActorIdentity { + return egress.ActorIdentity{ + Namespace: req.GetActorTemplateNamespace(), + Template: req.GetActorTemplateName(), + ActorID: req.GetActorId(), + Atespace: req.GetAtespace(), + } +} + +func ActorIdentityFromRestore(req *ateompb.RestoreWorkloadRequest) egress.ActorIdentity { + return egress.ActorIdentity{ + Namespace: req.GetActorTemplateNamespace(), + Template: req.GetActorTemplateName(), + ActorID: req.GetActorId(), + Atespace: req.GetAtespace(), + } +} + +func tcpRedirectExprs(sourceIP string, capturePort uint16) []expr.Any { + exprs := append(ipSourceEqual(sourceIP), + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: []byte{unix.IPPROTO_TCP}, + }, + ) + exprs = append(exprs, + &expr.Immediate{ + Register: 1, + Data: binaryutil.BigEndian.PutUint16(capturePort), + }, + &expr.Redir{ + RegisterProtoMin: 1, + }, + ) + return exprs +} + +func ipSourceEqual(ip string) []expr.Any { + return []expr.Any{ + &expr.Payload{ + DestRegister: 1, + Base: expr.PayloadBaseNetworkHeader, + Offset: 12, + Len: 4, + }, + &expr.Cmp{ + Op: expr.CmpOpEq, + Register: 1, + Data: net.ParseIP(ip).To4(), + }, + } +} + +func originalDestination(conn net.Conn) (net.Addr, error) { + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + return nil, fmt.Errorf("captured connection is %T, not *net.TCPConn", conn) + } + + rawConn, err := tcpConn.SyscallConn() + if err != nil { + return nil, err + } + + var addr *net.TCPAddr + var controlErr error + if err := rawConn.Control(func(fd uintptr) { + addr, controlErr = originalDstFromFD(int(fd)) + }); err != nil { + return nil, err + } + if controlErr != nil { + return nil, controlErr + } + return addr, nil +} + +func originalDstFromFD(fd int) (*net.TCPAddr, error) { + var raw unix.RawSockaddrInet4 + size := uint32(unsafe.Sizeof(raw)) + _, _, errno := unix.Syscall6( + unix.SYS_GETSOCKOPT, + uintptr(fd), + uintptr(unix.SOL_IP), + uintptr(unix.SO_ORIGINAL_DST), + uintptr(unsafe.Pointer(&raw)), + uintptr(unsafe.Pointer(&size)), + 0, + ) + if errno != 0 { + return nil, errno + } + if raw.Family != syscall.AF_INET { + return nil, fmt.Errorf("SO_ORIGINAL_DST returned address family %d", raw.Family) + } + return &net.TCPAddr{ + IP: net.IPv4(raw.Addr[0], raw.Addr[1], raw.Addr[2], raw.Addr[3]), + Port: int(binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&raw.Port))[:])), + }, nil +} diff --git a/internal/egress/capture.go b/internal/egress/capture.go new file mode 100644 index 000000000..1607e9933 --- /dev/null +++ b/internal/egress/capture.go @@ -0,0 +1,501 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/binary" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/http2" +) + +type ActorIdentity struct { + Namespace string + Template string + ActorID string + Atespace string + // TODO: Include worker_uid once egress identity is modeled as a signed + // first-class Substrate identity rather than plain actor metadata headers. +} + +type Config struct { + PEPAddress string + Listeners []Listener +} + +type Listener struct { + Port uint16 +} + +type OriginalDestinationFunc func(net.Conn) (net.Addr, error) + +type Capture struct { + cancel context.CancelFunc + listeners []net.Listener + transport *http2.Transport + wg sync.WaitGroup +} + +func ConfigForPEPAddress(pepAddress string, listeners []Listener) (Config, bool) { + pepAddress = strings.TrimSpace(pepAddress) + if pepAddress == "" { + return Config{}, false + } + return Config{PEPAddress: pepAddress, Listeners: listeners}, true +} + +func Start(ctx context.Context, identity ActorIdentity, cfg Config, originalDestination OriginalDestinationFunc) (*Capture, error) { + if originalDestination == nil { + return nil, errors.New("original destination resolver must be set") + } + + ctx, cancel := newCaptureContext(ctx) + // One HTTP/2 transport for the whole capture. The PEP address is fixed for + // the actor's lifetime, so CONNECT streams to the same destination authority + // multiplex over a pooled connection to the PEP instead of paying a fresh TCP + // dial + h2 handshake per captured connection. + capture := &Capture{cancel: cancel, transport: newPEPTransport(cfg.PEPAddress)} + for _, listenerCfg := range cfg.Listeners { + lis, err := net.Listen("tcp4", net.JoinHostPort("0.0.0.0", strconv.Itoa(int(listenerCfg.Port)))) + if err != nil { + capture.Close() + return nil, fmt.Errorf("while listening for captured egress on port %d: %w", listenerCfg.Port, err) + } + + capture.listeners = append(capture.listeners, lis) + capture.wg.Add(1) + go capture.serve(ctx, lis, identity, cfg.PEPAddress, originalDestination) + slog.InfoContext(ctx, "Started actor egress capture listener", + "port", listenerCfg.Port, + "pepAddress", cfg.PEPAddress) + } + return capture, nil +} + +// newPEPTransport builds the shared HTTP/2 transport whose connections all dial +// the fixed PEP address, regardless of the CONNECT authority. +func newPEPTransport(pepAddress string) *http2.Transport { + return &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, _ string, _ *tls.Config) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, network, pepAddress) + }, + } +} + +func newCaptureContext(ctx context.Context) (context.Context, context.CancelFunc) { + // The setup request context can be cancelled after the actor is running, but + // egress capture must keep serving until actor network cleanup closes it. + return context.WithCancel(context.WithoutCancel(ctx)) +} + +func (c *Capture) Close() error { + if c.cancel != nil { + c.cancel() + } + + var err error + for _, lis := range c.listeners { + if closeErr := lis.Close(); closeErr != nil && !errors.Is(closeErr, net.ErrClosed) { + err = errors.Join(err, closeErr) + } + } + c.wg.Wait() + if c.transport != nil { + c.transport.CloseIdleConnections() + } + return err +} + +func (c *Capture) serve(ctx context.Context, lis net.Listener, identity ActorIdentity, pepAddress string, originalDestination OriginalDestinationFunc) { + defer c.wg.Done() + for { + conn, err := lis.Accept() + if err != nil { + if ctx.Err() != nil || errors.Is(err, net.ErrClosed) { + return + } + slog.WarnContext(ctx, "Failed to accept captured egress connection", "err", err) + continue + } + c.wg.Add(1) + transport := c.transport + go func() { + defer c.wg.Done() + handleCapturedEgress(ctx, conn, identity, transport, pepAddress, originalDestination) + }() + } +} + +func handleCapturedEgress(ctx context.Context, actorConn net.Conn, identity ActorIdentity, transport *http2.Transport, pepAddress string, originalDestination OriginalDestinationFunc) { + stopActorClose := context.AfterFunc(ctx, func() { + _ = actorConn.Close() + }) + defer stopActorClose() + defer actorConn.Close() + + originalDst, err := originalDestination(actorConn) + if err != nil { + slog.WarnContext(ctx, "Failed to resolve captured egress original destination", "err", err) + return + } + + authority, initialBytes := deriveConnectAuthority(ctx, actorConn, originalDst) + tunnel, err := openCONNECTTunnel(ctx, transport, pepAddress, identity, originalDst, authority) + if err != nil { + slog.WarnContext(ctx, "Failed to open egress tunnel", + "originalDestination", originalDst.String(), + "connectAuthority", authority, + "err", err) + return + } + defer tunnel.Close() + + slog.InfoContext(ctx, "Proxying captured actor egress", + "actorID", identity.ActorID, + "actorTemplateNamespace", identity.Namespace, + "actorTemplateName", identity.Template, + "originalDestination", originalDst.String(), + "connectAuthority", authority) + + proxyByteStream(ctx, actorConn, tunnel, initialBytes) +} + +func proxyByteStream(ctx context.Context, actorConn net.Conn, tunnel io.ReadWriteCloser, initialBytes []byte) { + stopProxyClose := context.AfterFunc(ctx, func() { + _ = actorConn.Close() + _ = tunnel.Close() + }) + defer stopProxyClose() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + if len(initialBytes) > 0 { + if _, err := tunnel.Write(initialBytes); err != nil { + _ = tunnel.Close() + return + } + } + _, _ = io.Copy(tunnel, actorConn) + _ = tunnel.Close() + }() + go func() { + defer wg.Done() + _, _ = io.Copy(actorConn, tunnel) + if tcpConn, ok := actorConn.(*net.TCPConn); ok { + _ = tcpConn.CloseWrite() + } + }() + wg.Wait() +} + +func openCONNECTTunnel(ctx context.Context, transport *http2.Transport, pepAddress string, identity ActorIdentity, originalDst net.Addr, authority string) (io.ReadWriteCloser, error) { + // TODO: Add a transport selector here when there is a second supported + // egress tunnel protocol, such as TLS CONNECT or HBONE. + req, pr, pw := newConnectRequest(ctx, identity, originalDst, authority) + return roundTripConnect(transport, req, pr, pw, authority, pepAddress) +} + +func deriveConnectAuthority(ctx context.Context, actorConn net.Conn, originalDst net.Addr) (string, []byte) { + if tcpAddr, ok := originalDst.(*net.TCPAddr); ok { + return classifyConnectAuthority(ctx, actorConn, tcpAddr) + } + return originalDst.String(), nil +} + +// sniffReadTimeout bounds how long capture waits for the actor to send the +// bytes that reveal a CONNECT authority (TLS SNI or HTTP Host). The authority +// is derived from those bytes, so the tunnel cannot be opened until they +// arrive; a client that speaks only after the server (SMTP, some databases) +// waits out this deadline and then falls back to the original destination. +const sniffReadTimeout = 2 * time.Second + +const maxSniffBytes = 16 * 1024 + +func classifyConnectAuthority(ctx context.Context, actorConn net.Conn, originalDst *net.TCPAddr) (string, []byte) { + _ = actorConn.SetReadDeadline(time.Now().Add(sniffReadTimeout)) + defer actorConn.SetReadDeadline(time.Time{}) + + var initialBytes []byte + httpScanned := 0 // bytes already searched for the HTTP header terminator + buf := make([]byte, 2048) + for len(initialBytes) < maxSniffBytes { + n, err := actorConn.Read(buf) + if n > 0 { + initialBytes = append(initialBytes, buf[:n]...) + if initialBytes[0] == 0x16 { + // tlsClientHelloSNI is O(1) on an incomplete record (it checks + // the record length prefix before walking). + if sni, ok, needMore := tlsClientHelloSNI(initialBytes); ok { + return net.JoinHostPort(sni, strconv.Itoa(originalDst.Port)), initialBytes + } else if !needMore { + break + } + } else if httpHeadersComplete(initialBytes, &httpScanned) { + // Parse only once the full header block has arrived so a slow or + // byte-at-a-time sender does not re-parse the buffer each read. + if host, ok, _ := httpHostHeader(initialBytes); ok { + return authorityWithDefaultPort(host, originalDst.Port), initialBytes + } + break + } + } + if err != nil { + break + } + } + return originalDst.String(), initialBytes +} + +// httpHeadersComplete reports whether data contains the end-of-headers marker, +// searching only the bytes appended since the last call (with a small overlap +// for a marker split across reads) so the total scan cost stays linear in the +// sniffed byte count. It advances *scanned to the current length. +func httpHeadersComplete(data []byte, scanned *int) bool { + start := *scanned - 3 + if start < 0 { + start = 0 + } + if bytes.Contains(data[start:], []byte("\r\n\r\n")) || bytes.Contains(data[start:], []byte("\n\n")) { + return true + } + *scanned = len(data) + return false +} + +func httpHostHeader(data []byte) (string, bool, bool) { + // Scan for the end-of-headers marker on the raw bytes so an incomplete + // request does not copy the whole accumulated buffer into a string on every + // sniff read; only the bounded header block below is stringified. + headerEnd := bytes.Index(data, []byte("\r\n\r\n")) + separator := "\r\n" + if headerEnd == -1 { + headerEnd = bytes.Index(data, []byte("\n\n")) + separator = "\n" + } + if headerEnd == -1 { + return "", false, len(data) < maxSniffBytes + } + + lines := strings.Split(string(data[:headerEnd]), separator) + if len(lines) == 0 || !strings.Contains(lines[0], " ") { + return "", false, false + } + for _, line := range lines[1:] { + name, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + if strings.EqualFold(strings.TrimSpace(name), "host") { + host := strings.TrimSpace(value) + return host, host != "", false + } + } + return "", false, false +} + +func authorityWithDefaultPort(host string, port int) string { + host = strings.TrimSpace(host) + if host == "" { + return "" + } + if _, _, err := net.SplitHostPort(host); err == nil { + return host + } + return net.JoinHostPort(strings.Trim(host, "[]"), strconv.Itoa(port)) +} + +func tlsClientHelloSNI(data []byte) (string, bool, bool) { + if len(data) < 5 { + return "", false, true + } + if data[0] != 0x16 { + return "", false, false + } + recordLen := int(binary.BigEndian.Uint16(data[3:5])) + if len(data) < 5+recordLen { + return "", false, true + } + + record := data[5 : 5+recordLen] + if len(record) < 4 || record[0] != 0x01 { + return "", false, false + } + handshakeLen := int(record[1])<<16 | int(record[2])<<8 | int(record[3]) + if len(record) < 4+handshakeLen { + return "", false, false + } + clientHello := record[4 : 4+handshakeLen] + if len(clientHello) < 34 { + return "", false, false + } + + offset := 34 + if len(clientHello) < offset+1 { + return "", false, false + } + sessionIDLen := int(clientHello[offset]) + offset++ + if len(clientHello) < offset+sessionIDLen+2 { + return "", false, false + } + offset += sessionIDLen + + cipherSuitesLen := int(binary.BigEndian.Uint16(clientHello[offset : offset+2])) + offset += 2 + if len(clientHello) < offset+cipherSuitesLen+1 { + return "", false, false + } + offset += cipherSuitesLen + + compressionMethodsLen := int(clientHello[offset]) + offset++ + if len(clientHello) < offset+compressionMethodsLen+2 { + return "", false, false + } + offset += compressionMethodsLen + + extensionsLen := int(binary.BigEndian.Uint16(clientHello[offset : offset+2])) + offset += 2 + if len(clientHello) < offset+extensionsLen { + return "", false, false + } + extensions := clientHello[offset : offset+extensionsLen] + for len(extensions) >= 4 { + extensionType := binary.BigEndian.Uint16(extensions[0:2]) + extensionLen := int(binary.BigEndian.Uint16(extensions[2:4])) + extensions = extensions[4:] + if len(extensions) < extensionLen { + return "", false, false + } + extensionData := extensions[:extensionLen] + extensions = extensions[extensionLen:] + if extensionType != 0 { + continue + } + if len(extensionData) < 2 { + return "", false, false + } + serverNameListLen := int(binary.BigEndian.Uint16(extensionData[0:2])) + if len(extensionData) < 2+serverNameListLen { + return "", false, false + } + serverNames := extensionData[2 : 2+serverNameListLen] + for len(serverNames) >= 3 { + nameType := serverNames[0] + nameLen := int(binary.BigEndian.Uint16(serverNames[1:3])) + serverNames = serverNames[3:] + if len(serverNames) < nameLen { + return "", false, false + } + name := string(serverNames[:nameLen]) + serverNames = serverNames[nameLen:] + if nameType == 0 && name != "" { + return name, true, false + } + } + return "", false, false + } + return "", false, false +} + +func newConnectRequest(ctx context.Context, identity ActorIdentity, originalDst net.Addr, authority string) (*http.Request, *io.PipeReader, *io.PipeWriter) { + pr, pw := io.Pipe() + req := &http.Request{ + Method: http.MethodConnect, + URL: &url.URL{Scheme: "http", Host: authority}, + Host: authority, + Header: make(http.Header), + Body: pr, + ContentLength: -1, + } + req = req.WithContext(ctx) + // TODO: Replace these plain identity headers with a signed short-lived actor + // identity token for the PEP. The signed claims should include sub, aud, exp, + // iat, worker_uid, and the original destination so policy is evaluated over + // verified request identity rather than unsigned metadata. + req.Header.Set("x-ate-actor-id", identity.ActorID) + req.Header.Set("x-ate-atespace", identity.Atespace) + req.Header.Set("x-ate-actor-template", identity.Template) + req.Header.Set("x-ate-actor-template-namespace", identity.Namespace) + req.Header.Set("x-ate-original-destination", originalDst.String()) + if authority != originalDst.String() { + req.Header.Set("x-ate-connect-authority", authority) + } + return req, pr, pw +} + +func roundTripConnect( + transport *http2.Transport, + req *http.Request, + pr *io.PipeReader, + pw *io.PipeWriter, + connectAuthority string, + pepAddress string, +) (io.ReadWriteCloser, error) { + // The transport is shared across all captured connections, so failures close + // only this stream's pipes; idle connections to the PEP are reaped once in + // Capture.Close so unrelated in-flight streams keep multiplexing. + resp, err := transport.RoundTrip(req) + if err != nil { + _ = pr.CloseWithError(err) + _ = pw.CloseWithError(err) + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode > 299 { + _ = resp.Body.Close() + err := fmt.Errorf("CONNECT to %s through %s returned %s", connectAuthority, pepAddress, resp.Status) + _ = pr.CloseWithError(err) + _ = pw.CloseWithError(err) + return nil, err + } + return &connectStream{ + requestWriter: pw, + responseBody: resp.Body, + }, nil +} + +type connectStream struct { + requestWriter *io.PipeWriter + responseBody io.ReadCloser +} + +func (s *connectStream) Read(p []byte) (int, error) { + return s.responseBody.Read(p) +} + +func (s *connectStream) Write(p []byte) (int, error) { + return s.requestWriter.Write(p) +} + +func (s *connectStream) Close() error { + return errors.Join(s.requestWriter.Close(), s.responseBody.Close()) +} diff --git a/internal/egress/capture_test.go b/internal/egress/capture_test.go new file mode 100644 index 000000000..e48951d7d --- /dev/null +++ b/internal/egress/capture_test.go @@ -0,0 +1,318 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "context" + "crypto/tls" + "io" + "net" + "strings" + "testing" + "time" +) + +type contextKey string + +func TestNewCaptureContextIgnoresParentCancellation(t *testing.T) { + parent, parentCancel := context.WithCancel(context.WithValue(context.Background(), contextKey("trace"), "value")) + parentCancel() + + ctx, cancel := newCaptureContext(parent) + defer cancel() + + if err := ctx.Err(); err != nil { + t.Fatalf("capture context is cancelled by parent: %v", err) + } + if got := ctx.Value(contextKey("trace")); got != "value" { + t.Fatalf("capture context did not preserve values: got %v", got) + } + + cancel() + if err := ctx.Err(); err == nil { + t.Fatal("capture context was not cancelled by its own cancel func") + } +} + +func TestConfigForPEPAddress(t *testing.T) { + listeners := []Listener{{Port: 15001}} + cfg, ok := ConfigForPEPAddress("ate-egress.example:15008", listeners) + if !ok { + t.Fatal("ConfigForPEPAddress() ok = false, want true") + } + if cfg.PEPAddress != "ate-egress.example:15008" { + t.Fatalf("cfg.PEPAddress = %q, want ate-egress.example:15008", cfg.PEPAddress) + } + if len(cfg.Listeners) != 1 || cfg.Listeners[0].Port != 15001 { + t.Fatalf("cfg.Listeners = %+v, want port 15001", cfg.Listeners) + } +} + +func TestConfigForPEPAddressEmpty(t *testing.T) { + if _, ok := ConfigForPEPAddress("", nil); ok { + t.Fatal("ConfigForPEPAddress() ok = true, want false") + } +} + +func TestNewConnectRequestUsesConfiguredAuthority(t *testing.T) { + originalDst := &net.TCPAddr{IP: net.ParseIP("203.0.113.10"), Port: 443} + req, pr, pw := newConnectRequest(context.Background(), ActorIdentity{ + Namespace: "default", + Template: "counter", + ActorID: "my-counter-1", + Atespace: "team-a", + }, originalDst, "httpbin.org:443") + defer pr.Close() + defer pw.Close() + + if req.Host != "httpbin.org:443" { + t.Fatalf("req.Host = %q, want httpbin.org:443", req.Host) + } + if req.URL.Host != "httpbin.org:443" { + t.Fatalf("req.URL.Host = %q, want httpbin.org:443", req.URL.Host) + } + if got := req.Header.Get("x-ate-original-destination"); got != originalDst.String() { + t.Fatalf("x-ate-original-destination = %q, want %q", got, originalDst.String()) + } + if got := req.Header.Get("x-ate-connect-authority"); got != "httpbin.org:443" { + t.Fatalf("x-ate-connect-authority = %q, want httpbin.org:443", got) + } + if got := req.Header.Get("x-ate-atespace"); got != "team-a" { + t.Fatalf("x-ate-atespace = %q, want team-a", got) + } +} + +func TestDeriveConnectAuthorityFromTLSClientHelloSNI(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + + errCh := make(chan error, 1) + go func() { + tlsConn := tls.Client(clientConn, &tls.Config{ + ServerName: "httpbin.org", + InsecureSkipVerify: true, + }) + errCh <- tlsConn.Handshake() + }() + + originalDst := &net.TCPAddr{IP: net.ParseIP("203.0.113.10"), Port: 443} + authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, originalDst) + if authority != "httpbin.org:443" { + t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:443", authority) + } + if len(initialBytes) == 0 { + t.Fatal("deriveConnectAuthority() returned no initial bytes") + } + if _, ok, _ := tlsClientHelloSNI(initialBytes); !ok { + t.Fatal("initial bytes do not contain a parseable TLS ClientHello SNI") + } + + _ = clientConn.Close() + if err := <-errCh; err == nil { + t.Fatal("TLS handshake unexpectedly succeeded") + } else if err != io.ErrClosedPipe && !strings.Contains(err.Error(), "closed") { + t.Fatalf("TLS handshake error = %v, want closed connection", err) + } +} + +func TestDeriveConnectAuthorityFromTLSClientHelloSNIOnAnyPort(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + + errCh := make(chan error, 1) + go func() { + tlsConn := tls.Client(clientConn, &tls.Config{ + ServerName: "httpbin.org", + InsecureSkipVerify: true, + }) + errCh <- tlsConn.Handshake() + }() + + authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, &net.TCPAddr{ + IP: net.ParseIP("203.0.113.10"), + Port: 8443, + }) + if authority != "httpbin.org:8443" { + t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:8443", authority) + } + if len(initialBytes) == 0 { + t.Fatal("deriveConnectAuthority() returned no initial bytes") + } + + _ = clientConn.Close() + <-errCh +} + +func TestDeriveConnectAuthorityFromHTTPHost(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + + errCh := make(chan error, 1) + go func() { + _, err := clientConn.Write([]byte("GET /get HTTP/1.1\r\nHost: httpbin.org\r\nUser-Agent: test\r\n\r\n")) + errCh <- err + }() + + originalDst := &net.TCPAddr{IP: net.ParseIP("203.0.113.10"), Port: 80} + authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, originalDst) + if authority != "httpbin.org:80" { + t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:80", authority) + } + if string(initialBytes) != "GET /get HTTP/1.1\r\nHost: httpbin.org\r\nUser-Agent: test\r\n\r\n" { + t.Fatalf("initial bytes = %q", string(initialBytes)) + } + if err := <-errCh; err != nil { + t.Fatalf("client write returned error: %v", err) + } +} + +func TestDeriveConnectAuthorityFromHTTPHostOnAnyPort(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + + errCh := make(chan error, 1) + go func() { + _, err := clientConn.Write([]byte("GET /get HTTP/1.1\r\nHost: httpbin.org\r\n\r\n")) + errCh <- err + }() + + authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, &net.TCPAddr{ + IP: net.ParseIP("203.0.113.10"), + Port: 8080, + }) + if authority != "httpbin.org:8080" { + t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:8080", authority) + } + if string(initialBytes) != "GET /get HTTP/1.1\r\nHost: httpbin.org\r\n\r\n" { + t.Fatalf("initial bytes = %q", string(initialBytes)) + } + if err := <-errCh; err != nil { + t.Fatalf("client write returned error: %v", err) + } +} + +func TestDeriveConnectAuthorityFallsBackToOriginalDestination(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + + go func() { + _, _ = clientConn.Write([]byte("not http or tls")) + _ = clientConn.Close() + }() + + originalDst := &net.TCPAddr{IP: net.ParseIP("203.0.113.10"), Port: 2222} + authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, originalDst) + if authority != originalDst.String() { + t.Fatalf("deriveConnectAuthority() authority = %q, want %q", authority, originalDst.String()) + } + if string(initialBytes) != "not http or tls" { + t.Fatalf("initial bytes = %q", string(initialBytes)) + } +} + +func TestProxyByteStreamStopsWhenContextCancelled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + actorConn, actorPeer := net.Pipe() + defer actorPeer.Close() + tunnelConn, tunnelPeer := net.Pipe() + defer tunnelPeer.Close() + + done := make(chan struct{}) + go func() { + proxyByteStream(ctx, actorConn, tunnelConn, nil) + close(done) + }() + + cancel() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("proxyByteStream did not stop after context cancellation") + } +} + +func TestConnectStreamCloseClosesPipes(t *testing.T) { + pr, pw := io.Pipe() + defer pr.Close() + + stream := &connectStream{ + requestWriter: pw, + responseBody: io.NopCloser(strings.NewReader("")), + } + + if err := stream.Close(); err != nil { + t.Fatalf("stream.Close() returned error: %v", err) + } + // The request pipe is closed: a subsequent write fails. + if _, err := pw.Write([]byte("x")); err == nil { + t.Fatal("stream.Close() did not close the request writer") + } +} + +func TestHTTPHeadersCompleteDetectsMarkerSplitAcrossReads(t *testing.T) { + // The "\r\n\r\n" marker is delivered across two calls: "...\r\n\r" then "\n". + // The incremental search must still detect it via the small re-scan overlap. + scanned := 0 + first := []byte("GET / HTTP/1.1\r\nHost: example.com\r\n\r") + if httpHeadersComplete(first, &scanned) { + t.Fatal("httpHeadersComplete() = true on partial marker, want false") + } + if scanned != len(first) { + t.Fatalf("scanned = %d, want %d", scanned, len(first)) + } + full := append(first, '\n') + if !httpHeadersComplete(full, &scanned) { + t.Fatal("httpHeadersComplete() = false after marker completed, want true") + } +} + +func TestDeriveConnectAuthorityFromHTTPHostByteAtATime(t *testing.T) { + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + + request := "GET /get HTTP/1.1\r\nHost: httpbin.org\r\n\r\n" + go func() { + for i := 0; i < len(request); i++ { + if _, err := clientConn.Write([]byte{request[i]}); err != nil { + return + } + } + }() + + authority, _ := deriveConnectAuthority(context.Background(), serverConn, &net.TCPAddr{ + IP: net.ParseIP("203.0.113.10"), + Port: 80, + }) + if authority != "httpbin.org:80" { + t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:80", authority) + } +} + +func TestHTTPHostHeaderWithPort(t *testing.T) { + host, ok, needMore := httpHostHeader([]byte("GET / HTTP/1.1\r\nHost: example.com:8080\r\n\r\n")) + if !ok || needMore { + t.Fatalf("httpHostHeader() ok=%t needMore=%t, want ok=true needMore=false", ok, needMore) + } + if got := authorityWithDefaultPort(host, 80); got != "example.com:8080" { + t.Fatalf("authorityWithDefaultPort() = %q, want example.com:8080", got) + } +} diff --git a/internal/egress/env.go b/internal/egress/env.go new file mode 100644 index 000000000..65c7a3e13 --- /dev/null +++ b/internal/egress/env.go @@ -0,0 +1,32 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +const ( + // LabelUseEgressPEP is the consumer-side selector key (on an Actor or + // Atespace) that supplies the egress PEP address to use, as ":" + // (analogous to Istio ambient's istio.io/use-waypoint). Precedence is + // actor > atespace > global default (--default-egress-pep). ate-api uses the + // value directly and has no Gateway API dependency. + LabelUseEgressPEP = "ate.dev/use-egress-pep" +) + +const ( + DefaultCapturePort = uint16(15001) +) + +var DefaultCaptureListeners = []Listener{ + {Port: DefaultCapturePort}, +} diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index 4f51a5f2f..f631fa6c0 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -199,9 +199,10 @@ type RunRequest struct { // The sandbox binaries to use for booting this actor from scratch. atelet // fetches the relevant assets and records them with the actor's on-node state // so a later Checkpoint can pin the same version into the snapshot manifest. - SandboxAssets *SandboxAssets `protobuf:"bytes,7,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + SandboxAssets *SandboxAssets `protobuf:"bytes,7,opt,name=sandbox_assets,json=sandboxAssets,proto3" json:"sandbox_assets,omitempty"` + EgressPepAddress string `protobuf:"bytes,8,opt,name=egress_pep_address,json=egressPepAddress,proto3" json:"egress_pep_address,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RunRequest) Reset() { @@ -283,6 +284,13 @@ func (x *RunRequest) GetSandboxAssets() *SandboxAssets { return nil } +func (x *RunRequest) GetEgressPepAddress() string { + if x != nil { + return x.EgressPepAddress + } + return "" +} + // AssetFile is one content-addressed file atelet fetches for a sandbox runtime // (e.g. the gVisor runsc binary). type AssetFile struct { @@ -1253,9 +1261,10 @@ type RestoreRequest struct { // *RestoreRequest_ExternalConfig Config isRestoreRequest_Config `protobuf_oneof:"config"` // What content to restore from the checkpoint. - Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=atelet.SnapshotScope" json:"scope,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Scope SnapshotScope `protobuf:"varint,10,opt,name=scope,proto3,enum=atelet.SnapshotScope" json:"scope,omitempty"` + EgressPepAddress string `protobuf:"bytes,11,opt,name=egress_pep_address,json=egressPepAddress,proto3" json:"egress_pep_address,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreRequest) Reset() { @@ -1369,6 +1378,13 @@ func (x *RestoreRequest) GetScope() SnapshotScope { return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED } +func (x *RestoreRequest) GetEgressPepAddress() string { + if x != nil { + return x.EgressPepAddress + } + return "" +} + type isRestoreRequest_Config interface { isRestoreRequest_Config() } @@ -1425,7 +1441,7 @@ var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + "\n" + - "\fatelet.proto\x12\x06atelet\"\xc3\x02\n" + + "\fatelet.proto\x12\x06atelet\"\xf1\x02\n" + "\n" + "RunRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + @@ -1435,7 +1451,8 @@ const file_atelet_proto_rawDesc = "" + "\x18actor_template_namespace\x18\x04 \x01(\tR\x16actorTemplateNamespace\x12.\n" + "\x13actor_template_name\x18\x05 \x01(\tR\x11actorTemplateName\x12(\n" + "\x04spec\x18\x06 \x01(\v2\x14.atelet.WorkloadSpecR\x04spec\x12<\n" + - "\x0esandbox_assets\x18\a \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssets\"5\n" + + "\x0esandbox_assets\x18\a \x01(\v2\x15.atelet.SandboxAssetsR\rsandboxAssets\x12,\n" + + "\x12egress_pep_address\x18\b \x01(\tR\x10egressPepAddress\"5\n" + "\tAssetFile\x12\x10\n" + "\x03url\x18\x01 \x01(\tR\x03url\x12\x16\n" + "\x06sha256\x18\x02 \x01(\tR\x06sha256\"\x8e\x01\n" + @@ -1504,7 +1521,7 @@ const file_atelet_proto_rawDesc = "" + "\x05scope\x18\n" + " \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scopeB\b\n" + "\x06config\"\x14\n" + - "\x12CheckpointResponse\"\x8b\x04\n" + + "\x12CheckpointResponse\"\xb9\x04\n" + "\x0eRestoreRequest\x12(\n" + "\x10target_ateom_uid\x18\x01 \x01(\tR\x0etargetAteomUid\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -1517,7 +1534,8 @@ const file_atelet_proto_rawDesc = "" + "\flocal_config\x18\b \x01(\v2$.atelet.LocalCheckpointConfigurationH\x00R\vlocalConfig\x12R\n" + "\x0fexternal_config\x18\t \x01(\v2'.atelet.ExternalCheckpointConfigurationH\x00R\x0eexternalConfig\x12+\n" + "\x05scope\x18\n" + - " \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scopeB\b\n" + + " \x01(\x0e2\x15.atelet.SnapshotScopeR\x05scope\x12,\n" + + "\x12egress_pep_address\x18\v \x01(\tR\x10egressPepAddressB\b\n" + "\x06config\"\x11\n" + "\x0fRestoreResponse*F\n" + "\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 238032789..5030c560e 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -47,6 +47,8 @@ message RunRequest { // fetches the relevant assets and records them with the actor's on-node state // so a later Checkpoint can pin the same version into the snapshot manifest. SandboxAssets sandbox_assets = 7; + + string egress_pep_address = 8; } // AssetFile is one content-addressed file atelet fetches for a sandbox runtime @@ -228,6 +230,8 @@ message RestoreRequest { // What content to restore from the checkpoint. SnapshotScope scope = 10; + + string egress_pep_address = 11; } message RestoreResponse { diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index 2df3a1926..f1285d192 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -103,6 +103,7 @@ type RunWorkloadRequest struct { // to the local on-disk path atelet fetched it to (content-addressed, like // runsc_path). Empty for the gVisor runtime, which uses runsc_path. RuntimeAssetPaths map[string]string `protobuf:"bytes,7,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + EgressPepAddress string `protobuf:"bytes,8,opt,name=egress_pep_address,json=egressPepAddress,proto3" json:"egress_pep_address,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -186,6 +187,13 @@ func (x *RunWorkloadRequest) GetRuntimeAssetPaths() map[string]string { return nil } +func (x *RunWorkloadRequest) GetEgressPepAddress() string { + if x != nil { + return x.EgressPepAddress + } + return "" +} + // WorkloadSpec parallels Pod, but with far fewer configurable fields. type WorkloadSpec struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -392,6 +400,12 @@ func (x *HTTPGetAction) GetPort() int32 { return 0 } +// TODO: Add an EgressCaptureStatus ack (state enum with UNSPECIFIED=0 + +// requested/active PEP address) to RunWorkloadResponse/RestoreWorkloadResponse +// so ate-api persists the PEP the sandbox actually enforces instead of the one +// it requested. The zero-value enum makes pre-egress binaries detectable. +// Today a stale ateom image silently ignores egress_pep_address while actor +// status reports the binding. type RunWorkloadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -608,9 +622,10 @@ type RestoreWorkloadRequest struct { // atelet fetched it to (see RunWorkloadRequest). Empty for gVisor. RuntimeAssetPaths map[string]string `protobuf:"bytes,8,rep,name=runtime_asset_paths,json=runtimeAssetPaths,proto3" json:"runtime_asset_paths,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // What content to restore from the snapshot. - Scope SnapshotScope `protobuf:"varint,9,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Scope SnapshotScope `protobuf:"varint,9,opt,name=scope,proto3,enum=ateom.SnapshotScope" json:"scope,omitempty"` + EgressPepAddress string `protobuf:"bytes,10,opt,name=egress_pep_address,json=egressPepAddress,proto3" json:"egress_pep_address,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RestoreWorkloadRequest) Reset() { @@ -706,6 +721,13 @@ func (x *RestoreWorkloadRequest) GetScope() SnapshotScope { return SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED } +func (x *RestoreWorkloadRequest) GetEgressPepAddress() string { + if x != nil { + return x.EgressPepAddress + } + return "" +} + type RestoreWorkloadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -746,7 +768,7 @@ var File_ateom_proto protoreflect.FileDescriptor const file_ateom_proto_rawDesc = "" + "\n" + - "\vateom.proto\x12\x05ateom\"\xa9\x03\n" + + "\vateom.proto\x12\x05ateom\"\xd7\x03\n" + "\x12RunWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -756,7 +778,8 @@ const file_ateom_proto_rawDesc = "" + "\n" + "runsc_path\x18\x05 \x01(\tR\trunscPath\x12'\n" + "\x04spec\x18\x06 \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12`\n" + - "\x13runtime_asset_paths\x18\a \x03(\v20.ateom.RunWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x1aD\n" + + "\x13runtime_asset_paths\x18\a \x03(\v20.ateom.RunWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12,\n" + + "\x12egress_pep_address\x18\b \x01(\tR\x10egressPepAddress\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"@\n" + @@ -790,7 +813,7 @@ const file_ateom_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"C\n" + "\x1aCheckpointWorkloadResponse\x12%\n" + - "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\"\x8d\x04\n" + + "\x0esnapshot_files\x18\x01 \x03(\tR\rsnapshotFiles\"\xbb\x04\n" + "\x16RestoreWorkloadRequest\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x1d\n" + "\n" + @@ -802,7 +825,9 @@ const file_ateom_proto_rawDesc = "" + "\x04spec\x18\x06 \x01(\v2\x13.ateom.WorkloadSpecR\x04spec\x12.\n" + "\x13snapshot_uri_prefix\x18\a \x01(\tR\x11snapshotUriPrefix\x12d\n" + "\x13runtime_asset_paths\x18\b \x03(\v24.ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntryR\x11runtimeAssetPaths\x12*\n" + - "\x05scope\x18\t \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x1aD\n" + + "\x05scope\x18\t \x01(\x0e2\x14.ateom.SnapshotScopeR\x05scope\x12,\n" + + "\x12egress_pep_address\x18\n" + + " \x01(\tR\x10egressPepAddress\x1aD\n" + "\x16RuntimeAssetPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x19\n" + diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 619ff4f51..7ad85919b 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -63,6 +63,8 @@ message RunWorkloadRequest { // to the local on-disk path atelet fetched it to (content-addressed, like // runsc_path). Empty for the gVisor runtime, which uses runsc_path. map runtime_asset_paths = 7; + + string egress_pep_address = 8; } // WorkloadSpec parallels Pod, but with far fewer configurable fields. @@ -90,6 +92,12 @@ message HTTPGetAction { int32 port = 2; } +// TODO: Add an EgressCaptureStatus ack (state enum with UNSPECIFIED=0 + +// requested/active PEP address) to RunWorkloadResponse/RestoreWorkloadResponse +// so ate-api persists the PEP the sandbox actually enforces instead of the one +// it requested. The zero-value enum makes pre-egress binaries detectable. +// Today a stale ateom image silently ignores egress_pep_address while actor +// status reports the binding. message RunWorkloadResponse { } @@ -161,6 +169,8 @@ message RestoreWorkloadRequest { // What content to restore from the snapshot. SnapshotScope scope = 9; + + string egress_pep_address = 10; } message RestoreWorkloadResponse { diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 0b7b0c0ee..319a884a0 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -91,6 +91,11 @@ spec: - --session-id-jwt-pool=/run/session-id-jwt-pool/pool.json - --session-id-ca-pool=/run/session-id-ca-pool/pool.json - --workerpool-ca-certs=/run/workerpool-ca-certs/trust-bundle.pem + # Global-default egress PEP address (:). Empty when the + # ConfigMap does not set ATE_API_DEFAULT_EGRESS_PEP (non-egress + # installs), which leaves egress capture off unless a per-actor/atespace + # ate.dev/use-egress-pep selector matches. + - --default-egress-pep=@env env: - name: POD_NAME valueFrom: diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 0350f9fd9..1fb420ece 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -446,8 +446,20 @@ type Actor struct { // suspend/pause since eligibility is no longer a single fixed pool // reference on the ActorTemplate. WorkerPoolName string `protobuf:"bytes,12,opt,name=worker_pool_name,json=workerPoolName,proto3" json:"worker_pool_name,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The egress PEP address ate-api resolved for this actor on its most recent + // resume, in the form ..svc.cluster.local:. Empty + // means no PEP matched and egress capture is off. Set at worker assignment and + // cleared when the worker is released (suspend/pause). Records which PEP an + // actor is using so global-PEP membership is queryable without scanning logs. + EgressPepAddress string `protobuf:"bytes,13,opt,name=egress_pep_address,json=egressPepAddress,proto3" json:"egress_pep_address,omitempty"` + // Selector labels on the actor. The `ate.dev/use-egress-pep` key supplies the + // egress PEP address (as :) this actor should use, following + // the Istio ambient `istio.io/use-waypoint` model. This is the highest- + // precedence egress PEP selector (actor > atespace > global default). Set at + // create and mutable via UpdateActor; changes take effect on the next resume. + Labels map[string]string `protobuf:"bytes,14,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Actor) Reset() { @@ -564,12 +576,31 @@ func (x *Actor) GetWorkerPoolName() string { return "" } +func (x *Actor) GetEgressPepAddress() string { + if x != nil { + return x.EgressPepAddress + } + return "" +} + +func (x *Actor) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + // Atespace is the isolation boundary an Actor is created into. Global-scoped: // metadata.atespace is always empty; the atespace's identity is metadata.name. type Atespace struct { state protoimpl.MessageState `protogen:"open.v1"` // Common resource metadata: name, uid, version, timestamps. - Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // Selector labels on the atespace. The `ate.dev/use-egress-pep` key supplies + // the egress PEP address (as :) all actors in this atespace + // should use unless the actor sets its own. Medium-precedence egress PEP + // selector (actor > atespace > global default). + Labels map[string]string `protobuf:"bytes,2,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -611,6 +642,13 @@ func (x *Atespace) GetMetadata() *ResourceMetadata { return nil } +func (x *Atespace) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + // ObjectRef references a Substrate resource by its (atespace, name) identity. type ObjectRef struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -979,8 +1017,15 @@ type UpdateActorRequest struct { // worker_selector replaces the actor's current placement constraint. // Takes effect on the next ResumeActor call. WorkerSelector *Selector `protobuf:"bytes,2,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // labels is merged into the actor's selector labels: a key with a non-empty + // value is set (e.g. `ate.dev/use-egress-pep` with a : egress PEP + // address), a key with an empty value is deleted, and keys not present are + // left unchanged — an absent/empty map is a no-op, so callers unaware of + // labels cannot clear selectors set by others. Takes effect on the next + // ResumeActor call. + Labels map[string]string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateActorRequest) Reset() { @@ -1027,6 +1072,13 @@ func (x *UpdateActorRequest) GetWorkerSelector() *Selector { return nil } +func (x *UpdateActorRequest) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + type UpdateActorResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` @@ -2161,7 +2213,7 @@ const file_ateapi_proto_rawDesc = "" + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + "createTime\x12;\n" + "\vupdate_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "updateTime\"\x84\x06\n" + + "updateTime\"\xa0\a\n" + "\x05Actor\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x128\n" + "\x18actor_template_namespace\x18\x02 \x01(\tR\x16actorTemplateNamespace\x12.\n" + @@ -2176,7 +2228,12 @@ const file_ateapi_proto_rawDesc = "" + "\x14latest_snapshot_info\x18\n" + " \x01(\v2\x14.ateapi.SnapshotInfoR\x12latestSnapshotInfo\x129\n" + "\x0fworker_selector\x18\v \x01(\v2\x10.ateapi.SelectorR\x0eworkerSelector\x12(\n" + - "\x10worker_pool_name\x18\f \x01(\tR\x0eworkerPoolName\"\xb1\x01\n" + + "\x10worker_pool_name\x18\f \x01(\tR\x0eworkerPoolName\x12,\n" + + "\x12egress_pep_address\x18\r \x01(\tR\x10egressPepAddress\x121\n" + + "\x06labels\x18\x0e \x03(\v2\x19.ateapi.Actor.LabelsEntryR\x06labels\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb1\x01\n" + "\x06Status\x12\x16\n" + "\x12STATUS_UNSPECIFIED\x10\x00\x12\x13\n" + "\x0fSTATUS_RESUMING\x10\x01\x12\x12\n" + @@ -2185,9 +2242,13 @@ const file_ateapi_proto_rawDesc = "" + "\x10STATUS_SUSPENDED\x10\x04\x12\x12\n" + "\x0eSTATUS_PAUSING\x10\x05\x12\x11\n" + "\rSTATUS_PAUSED\x10\x06\x12\x12\n" + - "\x0eSTATUS_CRASHED\x10\a\"@\n" + + "\x0eSTATUS_CRASHED\x10\a\"\xb1\x01\n" + "\bAtespace\x124\n" + - "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\";\n" + + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x124\n" + + "\x06labels\x18\x02 \x03(\v2\x1c.ateapi.Atespace.LabelsEntryR\x06labels\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\";\n" + "\tObjectRef\x12\x1a\n" + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\"E\n" + @@ -2203,10 +2264,14 @@ const file_ateapi_proto_rawDesc = "" + "\x0fGetActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\"9\n" + "\x12CreateActorRequest\x12#\n" + - "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"x\n" + + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"\xf3\x01\n" + "\x12UpdateActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x129\n" + - "\x0fworker_selector\x18\x02 \x01(\v2\x10.ateapi.SelectorR\x0eworkerSelector\":\n" + + "\x0fworker_selector\x18\x02 \x01(\v2\x10.ateapi.SelectorR\x0eworkerSelector\x12>\n" + + "\x06labels\x18\x03 \x03(\v2&.ateapi.UpdateActorRequest.LabelsEntryR\x06labels\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\":\n" + "\x13UpdateActorResponse\x12#\n" + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\">\n" + "\x13SuspendActorRequest\x12'\n" + @@ -2315,7 +2380,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 42) var file_ateapi_proto_goTypes = []any{ (Actor_Status)(0), // 0: ateapi.Actor.Status (*ExternalSnapshotInfo)(nil), // 1: ateapi.ExternalSnapshotInfo @@ -2356,79 +2421,85 @@ var file_ateapi_proto_goTypes = []any{ (*MintCertRequest)(nil), // 36: ateapi.MintCertRequest (*MintCertResponse)(nil), // 37: ateapi.MintCertResponse nil, // 38: ateapi.Selector.MatchLabelsEntry - nil, // 39: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 40: google.protobuf.Timestamp + nil, // 39: ateapi.Actor.LabelsEntry + nil, // 40: ateapi.Atespace.LabelsEntry + nil, // 41: ateapi.UpdateActorRequest.LabelsEntry + nil, // 42: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 43: google.protobuf.Timestamp } var file_ateapi_proto_depIdxs = []int32{ 1, // 0: ateapi.SnapshotInfo.external:type_name -> ateapi.ExternalSnapshotInfo 2, // 1: ateapi.SnapshotInfo.local:type_name -> ateapi.LocalSnapshotInfo 38, // 2: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 40, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 40, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 43, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 43, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp 5, // 5: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 0, // 6: ateapi.Actor.status:type_name -> ateapi.Actor.Status 3, // 7: ateapi.Actor.latest_snapshot_info:type_name -> ateapi.SnapshotInfo 4, // 8: ateapi.Actor.worker_selector:type_name -> ateapi.Selector - 5, // 9: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata - 7, // 10: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace - 8, // 11: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 7, // 12: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace - 8, // 13: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef - 8, // 14: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef - 6, // 15: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor - 8, // 16: ateapi.UpdateActorRequest.actor:type_name -> ateapi.ObjectRef - 4, // 17: ateapi.UpdateActorRequest.worker_selector:type_name -> ateapi.Selector - 6, // 18: ateapi.UpdateActorResponse.actor:type_name -> ateapi.Actor - 8, // 19: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef - 6, // 20: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor - 8, // 21: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef - 6, // 22: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor - 8, // 23: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef - 6, // 24: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor - 8, // 25: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef - 29, // 26: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker - 6, // 27: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 30, // 28: ateapi.Worker.assignment:type_name -> ateapi.Assignment - 39, // 29: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 31, // 30: ateapi.Assignment.actor_template:type_name -> ateapi.KubeNamespacedObjectRef - 8, // 31: ateapi.Assignment.actor:type_name -> ateapi.ObjectRef - 14, // 32: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 15, // 33: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 16, // 34: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 18, // 35: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 20, // 36: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 22, // 37: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 24, // 38: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 25, // 39: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 27, // 40: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 9, // 41: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 10, // 42: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 11, // 43: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 13, // 44: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 32, // 45: ateapi.Control.DebugClear:input_type -> ateapi.DebugClearRequest - 34, // 46: ateapi.SessionIdentity.MintJWT:input_type -> ateapi.MintJWTRequest - 36, // 47: ateapi.SessionIdentity.MintCert:input_type -> ateapi.MintCertRequest - 6, // 48: ateapi.Control.GetActor:output_type -> ateapi.Actor - 6, // 49: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 17, // 50: ateapi.Control.UpdateActor:output_type -> ateapi.UpdateActorResponse - 19, // 51: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 21, // 52: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 23, // 53: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 6, // 54: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 26, // 55: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 28, // 56: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 7, // 57: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 7, // 58: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 12, // 59: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 7, // 60: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 33, // 61: ateapi.Control.DebugClear:output_type -> ateapi.DebugClearResponse - 35, // 62: ateapi.SessionIdentity.MintJWT:output_type -> ateapi.MintJWTResponse - 37, // 63: ateapi.SessionIdentity.MintCert:output_type -> ateapi.MintCertResponse - 48, // [48:64] is the sub-list for method output_type - 32, // [32:48] is the sub-list for method input_type - 32, // [32:32] is the sub-list for extension type_name - 32, // [32:32] is the sub-list for extension extendee - 0, // [0:32] is the sub-list for field type_name + 39, // 9: ateapi.Actor.labels:type_name -> ateapi.Actor.LabelsEntry + 5, // 10: ateapi.Atespace.metadata:type_name -> ateapi.ResourceMetadata + 40, // 11: ateapi.Atespace.labels:type_name -> ateapi.Atespace.LabelsEntry + 7, // 12: ateapi.CreateAtespaceRequest.atespace:type_name -> ateapi.Atespace + 8, // 13: ateapi.GetAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 7, // 14: ateapi.ListAtespacesResponse.atespaces:type_name -> ateapi.Atespace + 8, // 15: ateapi.DeleteAtespaceRequest.atespace:type_name -> ateapi.ObjectRef + 8, // 16: ateapi.GetActorRequest.actor:type_name -> ateapi.ObjectRef + 6, // 17: ateapi.CreateActorRequest.actor:type_name -> ateapi.Actor + 8, // 18: ateapi.UpdateActorRequest.actor:type_name -> ateapi.ObjectRef + 4, // 19: ateapi.UpdateActorRequest.worker_selector:type_name -> ateapi.Selector + 41, // 20: ateapi.UpdateActorRequest.labels:type_name -> ateapi.UpdateActorRequest.LabelsEntry + 6, // 21: ateapi.UpdateActorResponse.actor:type_name -> ateapi.Actor + 8, // 22: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef + 6, // 23: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor + 8, // 24: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef + 6, // 25: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor + 8, // 26: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef + 6, // 27: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor + 8, // 28: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef + 29, // 29: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 6, // 30: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 30, // 31: ateapi.Worker.assignment:type_name -> ateapi.Assignment + 42, // 32: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 31, // 33: ateapi.Assignment.actor_template:type_name -> ateapi.KubeNamespacedObjectRef + 8, // 34: ateapi.Assignment.actor:type_name -> ateapi.ObjectRef + 14, // 35: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 15, // 36: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 16, // 37: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 18, // 38: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 20, // 39: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 22, // 40: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 24, // 41: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 25, // 42: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 27, // 43: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 9, // 44: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 10, // 45: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 11, // 46: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 13, // 47: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 32, // 48: ateapi.Control.DebugClear:input_type -> ateapi.DebugClearRequest + 34, // 49: ateapi.SessionIdentity.MintJWT:input_type -> ateapi.MintJWTRequest + 36, // 50: ateapi.SessionIdentity.MintCert:input_type -> ateapi.MintCertRequest + 6, // 51: ateapi.Control.GetActor:output_type -> ateapi.Actor + 6, // 52: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 17, // 53: ateapi.Control.UpdateActor:output_type -> ateapi.UpdateActorResponse + 19, // 54: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 21, // 55: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 23, // 56: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 6, // 57: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 26, // 58: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 28, // 59: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 7, // 60: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 7, // 61: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 12, // 62: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 7, // 63: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 33, // 64: ateapi.Control.DebugClear:output_type -> ateapi.DebugClearResponse + 35, // 65: ateapi.SessionIdentity.MintJWT:output_type -> ateapi.MintJWTResponse + 37, // 66: ateapi.SessionIdentity.MintCert:output_type -> ateapi.MintCertResponse + 51, // [51:67] is the sub-list for method output_type + 35, // [35:51] is the sub-list for method input_type + 35, // [35:35] is the sub-list for extension type_name + 35, // [35:35] is the sub-list for extension extendee + 0, // [0:35] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -2446,7 +2517,7 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 1, - NumMessages: 39, + NumMessages: 42, NumExtensions: 0, NumServices: 2, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index 4c1d0461f..5deb206c3 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -157,6 +157,20 @@ message Actor { // suspend/pause since eligibility is no longer a single fixed pool // reference on the ActorTemplate. string worker_pool_name = 12; + + // The egress PEP address ate-api resolved for this actor on its most recent + // resume, in the form ..svc.cluster.local:. Empty + // means no PEP matched and egress capture is off. Set at worker assignment and + // cleared when the worker is released (suspend/pause). Records which PEP an + // actor is using so global-PEP membership is queryable without scanning logs. + string egress_pep_address = 13; + + // Selector labels on the actor. The `ate.dev/use-egress-pep` key supplies the + // egress PEP address (as :) this actor should use, following + // the Istio ambient `istio.io/use-waypoint` model. This is the highest- + // precedence egress PEP selector (actor > atespace > global default). Set at + // create and mutable via UpdateActor; changes take effect on the next resume. + map labels = 14; } // Atespace is the isolation boundary an Actor is created into. Global-scoped: @@ -164,6 +178,12 @@ message Actor { message Atespace { // Common resource metadata: name, uid, version, timestamps. ResourceMetadata metadata = 1; + + // Selector labels on the atespace. The `ate.dev/use-egress-pep` key supplies + // the egress PEP address (as :) all actors in this atespace + // should use unless the actor sets its own. Medium-precedence egress PEP + // selector (actor > atespace > global default). + map labels = 2; } // ObjectRef references a Substrate resource by its (atespace, name) identity. @@ -212,6 +232,14 @@ message UpdateActorRequest { // worker_selector replaces the actor's current placement constraint. // Takes effect on the next ResumeActor call. Selector worker_selector = 2; + + // labels is merged into the actor's selector labels: a key with a non-empty + // value is set (e.g. `ate.dev/use-egress-pep` with a : egress PEP + // address), a key with an empty value is deleted, and keys not present are + // left unchanged — an absent/empty map is a no-op, so callers unaware of + // labels cannot clear selectors set by others. Takes effect on the next + // ResumeActor call. + map labels = 3; } message UpdateActorResponse {