Skip to content

Commit a360515

Browse files
npolshakovanpolshakova
committed
fix atespace, classify port
Signed-off-by: npolshakova <nina.polshakova@solo.io>
1 parent 72f6342 commit a360515

7 files changed

Lines changed: 131 additions & 79 deletions

File tree

‎cmd/ateom-gvisor/egress_proxy.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,17 +48,22 @@ func (s *AteomService) startEgressCaptureIfEnabled(ctx context.Context, identity
4848
}
4949

5050
func addEgressCaptureRedirectRules(c *nftables.Conn, table *nftables.Table, prerouting *nftables.Chain, sourceIP string) {
51-
for _, redirect := range egress.DefaultCaptureRedirects {
52-
c.AddRule(&nftables.Rule{
53-
Table: table,
54-
Chain: prerouting,
55-
Exprs: tcpRedirectExprs(sourceIP, redirect.OriginalPort, redirect.CapturePort),
56-
})
57-
}
51+
c.AddRule(&nftables.Rule{
52+
Table: table,
53+
Chain: prerouting,
54+
Exprs: tcpRedirectExprs(sourceIP, egress.DefaultCapturePort),
55+
})
5856
}
5957

60-
func tcpRedirectExprs(sourceIP string, originalPort, capturePort uint16) []expr.Any {
61-
exprs := append(ipSourceEqual(sourceIP), tcpDestinationPortEqual(originalPort)...)
58+
func tcpRedirectExprs(sourceIP string, capturePort uint16) []expr.Any {
59+
exprs := append(ipSourceEqual(sourceIP),
60+
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
61+
&expr.Cmp{
62+
Op: expr.CmpOpEq,
63+
Register: 1,
64+
Data: []byte{unix.IPPROTO_TCP},
65+
},
66+
)
6267
exprs = append(exprs,
6368
&expr.Immediate{
6469
Register: 1,

‎cmd/ateom-microvm/egress_proxy.go‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,22 @@ func (s *AteomService) startEgressCaptureIfEnabled(ctx context.Context, identity
4949
}
5050

5151
func addEgressCaptureRedirectRules(c *nftables.Conn, table *nftables.Table, prerouting *nftables.Chain, sourceIP string) {
52-
for _, redirect := range egress.DefaultCaptureRedirects {
53-
c.AddRule(&nftables.Rule{
54-
Table: table,
55-
Chain: prerouting,
56-
Exprs: tcpRedirectExprs(sourceIP, redirect.OriginalPort, redirect.CapturePort),
57-
})
58-
}
52+
c.AddRule(&nftables.Rule{
53+
Table: table,
54+
Chain: prerouting,
55+
Exprs: tcpRedirectExprs(sourceIP, egress.DefaultCapturePort),
56+
})
5957
}
6058

61-
func tcpRedirectExprs(sourceIP string, originalPort, capturePort uint16) []expr.Any {
62-
exprs := append(ipSourceEqual(sourceIP), tcpDestinationPortEqual(originalPort)...)
59+
func tcpRedirectExprs(sourceIP string, capturePort uint16) []expr.Any {
60+
exprs := append(ipSourceEqual(sourceIP),
61+
&expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1},
62+
&expr.Cmp{
63+
Op: expr.CmpOpEq,
64+
Register: 1,
65+
Data: []byte{unix.IPPROTO_TCP},
66+
},
67+
)
6368
exprs = append(exprs,
6469
&expr.Immediate{
6570
Register: 1,

‎docs/egress-capture.md‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,8 @@ registry and snapshot bucket settings:
134134
Create an actor:
135135

136136
```bash
137-
kubectl ate create actor my-egress-1 --template ate-demo-egress/egress
137+
kubectl ate create atespace demo
138+
kubectl ate create actor my-egress-1 --template ate-demo-egress/egress -a demo
138139
```
139140

140141
Forward the router locally:
@@ -148,7 +149,7 @@ will make an outbound HTTPS request to `https://httpbin.org/get` by default:
148149

149150
```bash
150151
curl -i -X POST \
151-
-H "Host: my-egress-1.actors.resources.substrate.ate.dev" \
152+
-H "Host: my-egress-1.demo.actors.resources.substrate.ate.dev" \
152153
http://localhost:8000
153154
```
154155

@@ -168,7 +169,7 @@ To test a different `httpbin.org` path, pass it as the `url` query parameter:
168169

169170
```bash
170171
curl -i -X POST --get \
171-
-H "Host: my-egress-1.actors.resources.substrate.ate.dev" \
172+
-H "Host: my-egress-1.demo.actors.resources.substrate.ate.dev" \
172173
--data-urlencode "url=https://httpbin.org/headers" \
173174
"http://localhost:8000"
174175
```
@@ -182,7 +183,7 @@ agentgateway config only routes `httpbin.org:443`.
182183
Find the worker pod hosting the actor:
183184

184185
```bash
185-
actor_json=$(kubectl ate get actor my-egress-1 -o json)
186+
actor_json=$(kubectl ate get actor my-egress-1 -a demo -o json)
186187
ateom_ns=$(jq -r '.actors[0].ateomPodNamespace' <<<"${actor_json}")
187188
ateom_pod=$(jq -r '.actors[0].ateomPodName' <<<"${actor_json}")
188189

@@ -256,8 +257,8 @@ kubectl logs -n agentgateway-system deploy/agentgateway --tail=200
256257
## Clean up
257258

258259
```bash
259-
kubectl ate suspend actor my-egress-1
260-
kubectl ate delete actor my-egress-1
260+
kubectl ate suspend actor my-egress-1 -a demo
261+
kubectl ate delete actor my-egress-1 -a demo
261262
./hack/install-ate.sh --delete-demo-egress
262263
```
263264

@@ -286,7 +287,7 @@ If the capture listener logs are missing, confirm that the actor is running on a
286287
fresh worker pod created after egress was enabled:
287288

288289
```bash
289-
kubectl ate get actor my-egress-1
290+
kubectl ate get actor my-egress-1 -a demo
290291
kubectl get pods -n ate-demo-egress -l ate.dev/worker-pool=egress
291292
```
292293

‎hack/install-ate.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,8 @@ render_ate_system_manifests() {
169169
# Build everything resolved with base manifests for GKE
170170
run_ko resolve -f manifests/ate-install
171171
fi
172+
}
173+
172174
resolve_ipv4_addresses() {
173175
local host="$1"
174176
if command -v python3 >/dev/null 2>&1; then

‎internal/egress/capture.go‎

Lines changed: 17 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -219,59 +219,35 @@ func openCONNECTTunnel(ctx context.Context, pepAddress string, identity ActorIde
219219
}
220220

221221
func deriveConnectAuthority(ctx context.Context, actorConn net.Conn, originalDst net.Addr) (string, []byte) {
222-
if tcpAddr, ok := originalDst.(*net.TCPAddr); ok && tcpAddr.Port == 443 {
223-
authority, initialBytes := deriveTLSConnectAuthority(ctx, actorConn, tcpAddr)
224-
return authority, initialBytes
225-
}
226-
if tcpAddr, ok := originalDst.(*net.TCPAddr); ok && tcpAddr.Port == 80 {
227-
authority, initialBytes := deriveHTTPConnectAuthority(ctx, actorConn, tcpAddr)
228-
return authority, initialBytes
222+
if tcpAddr, ok := originalDst.(*net.TCPAddr); ok {
223+
return classifyConnectAuthority(ctx, actorConn, tcpAddr)
229224
}
230225
return originalDst.String(), nil
231226
}
232227

233-
func deriveTLSConnectAuthority(ctx context.Context, actorConn net.Conn, originalDst *net.TCPAddr) (string, []byte) {
234-
const maxClientHelloBytes = 16 * 1024
235-
_ = actorConn.SetReadDeadline(time.Now().Add(2 * time.Second))
236-
defer actorConn.SetReadDeadline(time.Time{})
237-
238-
var initialBytes []byte
239-
buf := make([]byte, 2048)
240-
for len(initialBytes) < maxClientHelloBytes {
241-
n, err := actorConn.Read(buf)
242-
if n > 0 {
243-
initialBytes = append(initialBytes, buf[:n]...)
244-
if sni, ok, needMore := tlsClientHelloSNI(initialBytes); ok {
245-
return net.JoinHostPort(sni, strconv.Itoa(originalDst.Port)), initialBytes
246-
} else if !needMore {
247-
break
248-
}
249-
}
250-
if err != nil {
251-
if ctx.Err() != nil {
252-
return originalDst.String(), initialBytes
253-
}
254-
break
255-
}
256-
}
257-
return originalDst.String(), initialBytes
258-
}
259-
260-
func deriveHTTPConnectAuthority(ctx context.Context, actorConn net.Conn, originalDst *net.TCPAddr) (string, []byte) {
261-
const maxHeaderBytes = 16 * 1024
228+
func classifyConnectAuthority(ctx context.Context, actorConn net.Conn, originalDst *net.TCPAddr) (string, []byte) {
229+
const maxSniffBytes = 16 * 1024
262230
_ = actorConn.SetReadDeadline(time.Now().Add(2 * time.Second))
263231
defer actorConn.SetReadDeadline(time.Time{})
264232

265233
var initialBytes []byte
266234
buf := make([]byte, 2048)
267-
for len(initialBytes) < maxHeaderBytes {
235+
for len(initialBytes) < maxSniffBytes {
268236
n, err := actorConn.Read(buf)
269237
if n > 0 {
270238
initialBytes = append(initialBytes, buf[:n]...)
271-
if host, ok, needMore := httpHostHeader(initialBytes); ok {
272-
return authorityWithDefaultPort(host, originalDst.Port), initialBytes
273-
} else if !needMore {
274-
break
239+
if initialBytes[0] == 0x16 {
240+
if sni, ok, needMore := tlsClientHelloSNI(initialBytes); ok {
241+
return net.JoinHostPort(sni, strconv.Itoa(originalDst.Port)), initialBytes
242+
} else if !needMore {
243+
break
244+
}
245+
} else {
246+
if host, ok, needMore := httpHostHeader(initialBytes); ok {
247+
return authorityWithDefaultPort(host, originalDst.Port), initialBytes
248+
} else if !needMore {
249+
break
250+
}
275251
}
276252
}
277253
if err != nil {

‎internal/egress/capture_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,35 @@ func TestDeriveConnectAuthorityFromTLSClientHelloSNI(t *testing.T) {
132132
}
133133
}
134134

135+
func TestDeriveConnectAuthorityFromTLSClientHelloSNIOnAnyPort(t *testing.T) {
136+
clientConn, serverConn := net.Pipe()
137+
defer clientConn.Close()
138+
defer serverConn.Close()
139+
140+
errCh := make(chan error, 1)
141+
go func() {
142+
tlsConn := tls.Client(clientConn, &tls.Config{
143+
ServerName: "httpbin.org",
144+
InsecureSkipVerify: true,
145+
})
146+
errCh <- tlsConn.Handshake()
147+
}()
148+
149+
authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, &net.TCPAddr{
150+
IP: net.ParseIP("203.0.113.10"),
151+
Port: 8443,
152+
})
153+
if authority != "httpbin.org:8443" {
154+
t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:8443", authority)
155+
}
156+
if len(initialBytes) == 0 {
157+
t.Fatal("deriveConnectAuthority() returned no initial bytes")
158+
}
159+
160+
_ = clientConn.Close()
161+
<-errCh
162+
}
163+
135164
func TestDeriveConnectAuthorityFromHTTPHost(t *testing.T) {
136165
clientConn, serverConn := net.Pipe()
137166
defer clientConn.Close()
@@ -156,6 +185,52 @@ func TestDeriveConnectAuthorityFromHTTPHost(t *testing.T) {
156185
}
157186
}
158187

188+
func TestDeriveConnectAuthorityFromHTTPHostOnAnyPort(t *testing.T) {
189+
clientConn, serverConn := net.Pipe()
190+
defer clientConn.Close()
191+
defer serverConn.Close()
192+
193+
errCh := make(chan error, 1)
194+
go func() {
195+
_, err := clientConn.Write([]byte("GET /get HTTP/1.1\r\nHost: httpbin.org\r\n\r\n"))
196+
errCh <- err
197+
}()
198+
199+
authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, &net.TCPAddr{
200+
IP: net.ParseIP("203.0.113.10"),
201+
Port: 8080,
202+
})
203+
if authority != "httpbin.org:8080" {
204+
t.Fatalf("deriveConnectAuthority() authority = %q, want httpbin.org:8080", authority)
205+
}
206+
if string(initialBytes) != "GET /get HTTP/1.1\r\nHost: httpbin.org\r\n\r\n" {
207+
t.Fatalf("initial bytes = %q", string(initialBytes))
208+
}
209+
if err := <-errCh; err != nil {
210+
t.Fatalf("client write returned error: %v", err)
211+
}
212+
}
213+
214+
func TestDeriveConnectAuthorityFallsBackToOriginalDestination(t *testing.T) {
215+
clientConn, serverConn := net.Pipe()
216+
defer clientConn.Close()
217+
defer serverConn.Close()
218+
219+
go func() {
220+
_, _ = clientConn.Write([]byte("not http or tls"))
221+
_ = clientConn.Close()
222+
}()
223+
224+
originalDst := &net.TCPAddr{IP: net.ParseIP("203.0.113.10"), Port: 2222}
225+
authority, initialBytes := deriveConnectAuthority(context.Background(), serverConn, originalDst)
226+
if authority != originalDst.String() {
227+
t.Fatalf("deriveConnectAuthority() authority = %q, want %q", authority, originalDst.String())
228+
}
229+
if string(initialBytes) != "not http or tls" {
230+
t.Fatalf("initial bytes = %q", string(initialBytes))
231+
}
232+
}
233+
159234
func TestProxyByteStreamStopsWhenContextCancelled(t *testing.T) {
160235
ctx, cancel := context.WithCancel(context.Background())
161236
actorConn, actorPeer := net.Pipe()

‎internal/egress/env.go‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,9 @@ const (
2020
)
2121

2222
const (
23-
DefaultCapturePort = uint16(15001)
24-
DefaultOriginalHTTPPort = uint16(80)
25-
DefaultOriginalTLSPort = uint16(443)
23+
DefaultCapturePort = uint16(15001)
2624
)
2725

28-
type Redirect struct {
29-
OriginalPort uint16
30-
CapturePort uint16
31-
}
32-
33-
var DefaultCaptureRedirects = []Redirect{
34-
{OriginalPort: DefaultOriginalHTTPPort, CapturePort: DefaultCapturePort},
35-
{OriginalPort: DefaultOriginalTLSPort, CapturePort: DefaultCapturePort},
36-
}
37-
3826
var DefaultCaptureListeners = []Listener{
3927
{Port: DefaultCapturePort},
4028
}

0 commit comments

Comments
 (0)