Skip to content

Commit 99be1f6

Browse files
EItanyahowardjohnpeterj
committed
router: add pluggable networking providers (#10)
* router: add pluggable networking providers * make websockets work in agw, make agw default in install script Signed-off-by: Peter Jausovec <peter.jausovec@solo.io> --------- Signed-off-by: Peter Jausovec <peter.jausovec@solo.io> Co-authored-by: John Howard <john.howard@solo.io> Co-authored-by: Peter Jausovec <peter.jausovec@solo.io>
1 parent e059406 commit 99be1f6

18 files changed

Lines changed: 1073 additions & 378 deletions

File tree

cmd/atenet/internal/router.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,15 @@ func NewRouterCmd() *cobra.Command {
5050
cmd.Flags().IntVar(&cfg.XdsPort, "port-xds", 18000, "TCP port listening for the xDS dynamic Envoy connections")
5151
cmd.Flags().IntVar(&cfg.ExtprocPort, "port-extproc", 50051, "Listen port for the Envoy dynamic External Processing (ext_proc) server")
5252
cmd.Flags().StringVar(&cfg.ExtprocAddr, "extproc-address", "127.0.0.1", "Host IP or address of the Envoy External Processing (ext_proc) server")
53+
cmd.Flags().StringVar(&cfg.NetworkingMode, "networking-mode", router.NetworkingModeEnvoy, "Networking proxy mode: envoy or agentgateway")
5354
cmd.Flags().StringVar(&cfg.EnvoyImage, "envoy-image", "envoyproxy/envoy:v1.30-latest", "Image URI used for dynamically launched router instances")
55+
cmd.Flags().StringVar(&cfg.AgentgatewayImage, "agentgateway-image", "cr.agentgateway.dev/agentgateway:v1.3.0-alpha.1", "Image URI used for Agentgateway router instances")
5456
cmd.Flags().StringVar(&cfg.TemplatesFile, "actor-templates-file", "", "Path to offline YAML configuration file listing ActorTemplates")
5557
cmd.Flags().IntVar(&cfg.StatusPort, "status-port", 4040, "Port to serve /statusz on (set <= 0 to disable serving status)")
5658
cmd.Flags().DurationVar(&cfg.HealthInterval, "health-interval", 1*time.Second, "Interval for checking health of dependent services")
5759
cmd.Flags().IntVar(&cfg.HttpsPort, "port-https", 8443, "TCP port for HTTPS workload traffic entering through the Envoy Router")
58-
cmd.Flags().StringVar(&cfg.EnvoyCertPath, "envoy-cert-path", "", "Path to the Envoy certificate file (if empty, a self-signed cert will be generated for testing)")
60+
cmd.Flags().StringVar(&cfg.TLSCertPath, "tls-cert-path", "", "Path to the proxy TLS certificate file")
61+
cmd.Flags().StringVar(&cfg.TLSKeyPath, "tls-key-path", "", "Path to the proxy TLS private key file")
5962
cmd.Flags().StringVar(&cfg.OtlpCollectorAddress, "otlp-collector-address", "", "host:port of the OTLP gRPC collector that Envoy reports tracing spans to (empty disables Envoy tracing)")
6063

6164
return cmd
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package router
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"strings"
21+
22+
corev1 "k8s.io/api/core/v1"
23+
"k8s.io/apimachinery/pkg/util/intstr"
24+
)
25+
26+
type agentgatewayProvider struct {
27+
cfg RouterConfig
28+
}
29+
30+
func (p agentgatewayProvider) Name() string {
31+
return NetworkingModeAgentgateway
32+
}
33+
34+
func (p agentgatewayProvider) RequiresXDS() bool {
35+
return false
36+
}
37+
38+
func (p agentgatewayProvider) ConfigMapData() map[string]string {
39+
return map[string]string{"config.yaml": p.localConfig()}
40+
}
41+
42+
func (p agentgatewayProvider) Container() corev1.Container {
43+
ports := []corev1.ContainerPort{
44+
{Name: "http", ContainerPort: int32(p.cfg.HttpPort)},
45+
{Name: "readiness", ContainerPort: 15021},
46+
{Name: "metrics", ContainerPort: 15020},
47+
}
48+
if p.cfg.HttpsPort > 0 && tlsCertPath(p.cfg) != "" {
49+
ports = append(ports, corev1.ContainerPort{Name: "https", ContainerPort: int32(p.cfg.HttpsPort)})
50+
}
51+
52+
return corev1.Container{
53+
Name: "agentgateway",
54+
Image: p.cfg.AgentgatewayImage,
55+
Args: []string{"-f", "/etc/agentgateway/config.yaml"},
56+
Ports: ports,
57+
VolumeMounts: []corev1.VolumeMount{
58+
{Name: "proxy-config", MountPath: "/etc/agentgateway"},
59+
},
60+
ReadinessProbe: &corev1.Probe{
61+
ProbeHandler: corev1.ProbeHandler{
62+
HTTPGet: &corev1.HTTPGetAction{
63+
Path: "/healthz/ready",
64+
Port: intstr.FromInt32(15021),
65+
},
66+
},
67+
PeriodSeconds: 10,
68+
},
69+
}
70+
}
71+
72+
func (p agentgatewayProvider) ServicePorts() []corev1.ServicePort {
73+
ports := []corev1.ServicePort{
74+
{Name: "http", Port: int32(p.cfg.HttpPort), TargetPort: intstr.FromString("http")},
75+
}
76+
if p.cfg.HttpsPort > 0 && tlsCertPath(p.cfg) != "" {
77+
ports = append(ports, corev1.ServicePort{Name: "https", Port: int32(p.cfg.HttpsPort), TargetPort: intstr.FromString("https")})
78+
}
79+
return ports
80+
}
81+
82+
func (p agentgatewayProvider) CheckReady(ctx context.Context) (bool, string) {
83+
return checkHTTPReady(ctx, "http://127.0.0.1:15021/healthz/ready", "")
84+
}
85+
86+
func (p agentgatewayProvider) localConfig() string {
87+
httpRoute := p.routeBlock("substrate-http")
88+
config := fmt.Sprintf(`# yaml-language-server: $schema=https://agentgateway.dev/schema/config
89+
config:
90+
adminAddr: "127.0.0.1:15000"
91+
readinessAddr: "0.0.0.0:15021"
92+
statsAddr: "0.0.0.0:15020"
93+
binds:
94+
- port: %d
95+
listeners:
96+
- name: http
97+
protocol: HTTP
98+
routes:
99+
%s`, p.cfg.HttpPort, indent(httpRoute, 4))
100+
101+
if p.cfg.HttpsPort > 0 && tlsCertPath(p.cfg) != "" {
102+
cert := tlsCertPath(p.cfg)
103+
key := tlsKeyPath(p.cfg)
104+
config += fmt.Sprintf(`- port: %d
105+
listeners:
106+
- name: https
107+
protocol: HTTPS
108+
tls:
109+
cert: %q
110+
key: %q
111+
routes:
112+
%s`, p.cfg.HttpsPort, cert, key, indent(p.routeBlock("substrate-https"), 4))
113+
}
114+
115+
return config
116+
}
117+
118+
func (p agentgatewayProvider) routeBlock(name string) string {
119+
extprocHost := fmt.Sprintf("%s:%d", p.cfg.ExtprocAddr, p.cfg.ExtprocPort)
120+
// processingOptions limit ext_proc to request headers only; agentgateway defaults
121+
// break WebSocket upgrades because this server only handles headers.
122+
return fmt.Sprintf(`- name: %s
123+
matches:
124+
- path:
125+
pathPrefix: /
126+
policies:
127+
extProc:
128+
host: %q
129+
failureMode: failClosed
130+
processingOptions:
131+
requestBodyMode: none
132+
responseBodyMode: none
133+
requestHeaderMode: send
134+
responseHeaderMode: skip
135+
requestTrailerMode: skip
136+
responseTrailerMode: skip
137+
backends:
138+
- dynamic: {}
139+
`, name, extprocHost)
140+
}
141+
142+
func indent(s string, spaces int) string {
143+
prefix := strings.Repeat(" ", spaces)
144+
lines := strings.Split(s, "\n")
145+
for i, line := range lines {
146+
if line != "" {
147+
lines[i] = prefix + line
148+
}
149+
}
150+
return strings.Join(lines, "\n")
151+
}

cmd/atenet/internal/router/controller.go

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@ type Controller struct {
3131
cfg RouterConfig
3232
xdsSrv *XdsServer
3333
extprocSrv *ExtProcServer
34+
provider proxyProvider
3435

3536
atStore atStore
36-
envoyRunner *envoyrunner
37+
proxyRunner *proxyrunner
3738
}
3839

3940
func NewController(
@@ -42,8 +43,11 @@ func NewController(
4243
cfg RouterConfig,
4344
xdsSrv *XdsServer,
4445
extprocSrv *ExtProcServer,
46+
provider proxyProvider,
4547
) *Controller {
46-
xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr)
48+
if xdsSrv != nil {
49+
xdsSrv.SetConfig(cfg.HttpPort, cfg.ExtprocPort, cfg.ExtprocAddr)
50+
}
4751

4852
var store atStore
4953
if cfg.TemplatesFile != "" {
@@ -58,9 +62,10 @@ func NewController(
5862
cfg: cfg,
5963
xdsSrv: xdsSrv,
6064
extprocSrv: extprocSrv,
65+
provider: provider,
6166

6267
atStore: store,
63-
envoyRunner: newEnvoyRunner(k8sClient, cfg),
68+
proxyRunner: newProxyRunner(k8sClient, cfg, provider),
6469
}
6570
}
6671

@@ -92,16 +97,18 @@ func (c *Controller) reconcile(ctx context.Context) error {
9297
return err
9398
}
9499

95-
if err := c.xdsSrv.UpdateSnapshot(); err != nil {
96-
slog.ErrorContext(ctx, "xDS Configuration generation problem", slog.String("err", err.Error()))
97-
return err
100+
if c.provider.RequiresXDS() {
101+
if err := c.xdsSrv.UpdateSnapshot(); err != nil {
102+
slog.ErrorContext(ctx, "xDS Configuration generation problem", slog.String("err", err.Error()))
103+
return err
104+
}
98105
}
99106

100107
if !c.cfg.Standalone && c.cfg.TemplatesFile == "" {
101-
// Reconcile Envoy router Deployment and Kubernetes cluster entities
102-
err := c.envoyRunner.reconcile(ctx)
108+
// Reconcile router proxy Deployment and Kubernetes cluster entities.
109+
err := c.proxyRunner.reconcile(ctx)
103110
if err != nil {
104-
slog.ErrorContext(ctx, "Error during Envoy router reconciliation", slog.String("err", err.Error()))
111+
slog.ErrorContext(ctx, "Error during router proxy reconciliation", slog.String("err", err.Error()))
105112
return err
106113
}
107114
}

cmd/atenet/internal/router/dashboard.html

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -252,12 +252,16 @@ <h1>atenet Router Status</h1>
252252
<span class="label">Namespace Context</span>
253253
<span class="value">{{ .Namespace }}</span>
254254
</div>
255+
<div class="metadata-item">
256+
<span class="label">Networking Mode</span>
257+
<span class="value">{{ .NetworkingMode }}</span>
258+
</div>
255259
</div>
256260

257261
<div class="card">
258262
<div class="card-title">Component Network Allocation</div>
259263
<div class="metadata-item">
260-
<span class="label">Workload Port (Http Envoy)</span>
264+
<span class="label">Workload HTTP Port</span>
261265
<span class="value">{{ .HttpPort }}</span>
262266
</div>
263267
<div class="metadata-item">
@@ -278,20 +282,20 @@ <h1>atenet Router Status</h1>
278282
<div class="card-title">System Component Health Checks</div>
279283

280284
<div class="metadata-item">
281-
<span class="label">Envoy Health</span>
282-
{{ if .Health.Envoy.Healthy }}
285+
<span class="label">Proxy Health ({{ .Health.Provider }})</span>
286+
{{ if .Health.Proxy.Healthy }}
283287
<span class="value badge" style="background: rgba(16, 185, 129, 0.1); color: #10b981;">Healthy</span>
284288
{{ else }}
285289
<span class="value badge" style="background: rgba(239, 68, 68, 0.1); color: #ef4444;">Degraded</span>
286290
{{ end }}
287291
</div>
288292
<div class="metadata-item" style="margin-top: -0.25rem; margin-bottom: 0.5rem; font-size: 0.8rem;">
289293
<span class="label">Message / Err:</span>
290-
<span class="value" style="color: var(--text-secondary);">{{ .Health.Envoy.Message }}</span>
294+
<span class="value" style="color: var(--text-secondary);">{{ .Health.Proxy.Message }}</span>
291295
</div>
292296
<div class="metadata-item" style="font-size: 0.75rem; color: var(--text-secondary);">
293-
<span>Ok: {{ .Health.Envoy.SuccessCount }} | Err: {{ .Health.Envoy.FailureCount }}</span>
294-
<span>LKG: {{ if not .Health.Envoy.LastSuccess.IsZero }}{{ .Health.Envoy.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }}</span>
297+
<span>Ok: {{ .Health.Proxy.SuccessCount }} | Err: {{ .Health.Proxy.FailureCount }}</span>
298+
<span>LKG: {{ if not .Health.Proxy.LastSuccess.IsZero }}{{ .Health.Proxy.LastSuccess.Format "15:04:05" }}{{ else }}N/A{{ end }}</span>
295299
</div>
296300

297301
<div style="border-bottom: 1px solid var(--card-border); margin: 0.5rem 0;"></div>
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package router
16+
17+
import (
18+
"context"
19+
"fmt"
20+
21+
corev1 "k8s.io/api/core/v1"
22+
"k8s.io/apimachinery/pkg/util/intstr"
23+
)
24+
25+
type envoyProvider struct {
26+
cfg RouterConfig
27+
}
28+
29+
func (p envoyProvider) Name() string {
30+
return NetworkingModeEnvoy
31+
}
32+
33+
func (p envoyProvider) RequiresXDS() bool {
34+
return true
35+
}
36+
37+
func (p envoyProvider) ConfigMapData() map[string]string {
38+
envoyYaml := fmt.Sprintf(`admin:
39+
address:
40+
socket_address:
41+
address: 0.0.0.0
42+
port_value: 9901
43+
44+
node:
45+
id: %s
46+
cluster: substrate-router-cluster
47+
48+
dynamic_resources:
49+
lds_config:
50+
resource_api_version: V3
51+
ads: {}
52+
cds_config:
53+
resource_api_version: V3
54+
ads: {}
55+
ads_config:
56+
api_type: GRPC
57+
transport_api_version: V3
58+
grpc_services:
59+
- envoy_grpc:
60+
cluster_name: xds_cluster
61+
62+
static_resources:
63+
clusters:
64+
- name: xds_cluster
65+
connect_timeout: 0.25s
66+
type: STRICT_DNS
67+
lb_policy: ROUND_ROBIN
68+
typed_extension_protocol_options:
69+
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
70+
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
71+
explicit_http_config:
72+
http2_protocol_options: {}
73+
load_assignment:
74+
cluster_name: xds_cluster
75+
endpoints:
76+
- lb_endpoints:
77+
- endpoint:
78+
address:
79+
socket_address:
80+
address: atenet-router
81+
port_value: %d
82+
`, NodeID, p.cfg.XdsPort)
83+
84+
return map[string]string{"envoy.yaml": envoyYaml}
85+
}
86+
87+
func (p envoyProvider) Container() corev1.Container {
88+
return corev1.Container{
89+
Name: "envoy",
90+
Image: p.cfg.EnvoyImage,
91+
Command: []string{
92+
"/usr/local/bin/envoy",
93+
"-c",
94+
"/etc/envoy/envoy.yaml",
95+
"--component-log-level",
96+
"upstream:debug,router:debug,ext_proc:debug",
97+
},
98+
Ports: []corev1.ContainerPort{
99+
{Name: "http", ContainerPort: int32(p.cfg.HttpPort)},
100+
{Name: "https", ContainerPort: int32(p.cfg.HttpsPort)},
101+
{Name: "admin", ContainerPort: 9901},
102+
},
103+
VolumeMounts: []corev1.VolumeMount{
104+
{Name: "proxy-config", MountPath: "/etc/envoy"},
105+
},
106+
}
107+
}
108+
109+
func (p envoyProvider) ServicePorts() []corev1.ServicePort {
110+
return []corev1.ServicePort{
111+
{Name: "http", Port: int32(p.cfg.HttpPort), TargetPort: intstr.FromString("http")},
112+
{Name: "https", Port: int32(p.cfg.HttpsPort), TargetPort: intstr.FromString("https")},
113+
}
114+
}
115+
116+
func (p envoyProvider) CheckReady(ctx context.Context) (bool, string) {
117+
return checkHTTPReady(ctx, "http://127.0.0.1:9901/ready", "LIVE")
118+
}

0 commit comments

Comments
 (0)