Skip to content

Commit b2c6a8c

Browse files
hweawerhweawer
committed
Add nginx timeouts
1 parent 58bfdd6 commit b2c6a8c

11 files changed

Lines changed: 210 additions & 6 deletions

File tree

‎agent/cmd/cmd.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -390,8 +390,12 @@ func (a *App) Run(ctx context.Context) error {
390390
"registry_server": nginx.GetServer(
391391
a.config.Registry.Docker.HTTP.Net, a.config.Registry.Docker.HTTP.Addr),
392392
"agent_server": fmt.Sprintf("127.0.0.1:%d", a.flags.AgentServerPort),
393-
"registry_backup": a.config.RegistryBackup},
394-
nginx.WithTLS(a.config.TLS))
393+
"registry_backup": a.config.RegistryBackup,
394+
// Pass timeout parameters from agent server config
395+
"download_timeout": nginx.FormatDurationForNginx(a.config.AgentServer.DownloadTimeout),
396+
"container_runtime_timeout": nginx.FormatDurationForNginx(a.config.AgentServer.ContainerRuntimeTimeout),
397+
"readiness_timeout": nginx.FormatDurationForNginx(a.config.AgentServer.ReadinessTimeout),
398+
}, nginx.WithTLS(a.config.TLS))
395399
nginxDone <- err
396400
}()
397401

‎config/agent/base.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,16 @@ registry:
5858

5959
peer_id_factory: addr_hash
6060

61+
agentserver:
62+
# Timeout configurations (also used by nginx)
63+
download_timeout: 5m # nginx proxy_read_timeout for downloads
64+
container_runtime_timeout: 10m # nginx timeout for container operations (pull/preload)
65+
readiness_timeout: 30s # nginx timeout for health checks
66+
67+
# Request configuration
68+
max_request_body_size: 1MB # Maximum size for patch requests
69+
enable_request_logging: false # Enable detailed request logging
70+
6171
# Allow agent to only serve localhost and Docker default bridge requests.
6272
allowed_cidrs:
6373
- 127.0.0.1

‎config/origin/base.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,17 @@ blobserver:
4848
listener:
4949
net: unix
5050
addr: /tmp/kraken-origin.sock
51+
52+
# Timeout configurations (also used by nginx)
53+
download_timeout: 5m # nginx proxy_read_timeout for downloads
54+
upload_timeout: 10m # nginx proxy_read_timeout/send_timeout for uploads
55+
replication_timeout: 3m # nginx timeout for replication operations
56+
backend_timeout: 2m # nginx proxy_connect_timeout
57+
readiness_timeout: 30s # internal readiness check timeout
58+
59+
# Concurrency limits
60+
max_concurrent_downloads: 10
61+
max_concurrent_uploads: 5
5162

5263
nginx:
5364
name: kraken-origin

‎config/tracker/base.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,16 @@ trackerserver:
4040
listener:
4141
net: unix
4242
addr: /tmp/kraken-tracker.sock
43+
44+
# Timeout configurations (also used by nginx)
45+
metainfo_timeout: 2m # nginx proxy_read_timeout for metainfo requests to origins
46+
announce_timeout: 30s # nginx proxy_read_timeout for announce operations
47+
readiness_timeout: 30s # nginx timeout for health checks
48+
49+
# Rate limiting
50+
get_metainfo_limit: 1s # Limits unique metainfo requests per namespace/digest
51+
announce_limit: 50 # Maximum peers returned on each announce
52+
announce_interval: 3s # How often peers should announce
4353

4454
nginx:
4555
name: kraken-tracker

‎nginx/config/agent.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,44 @@ server {
4040
gzip on;
4141
gzip_types text/plain test/csv application/json;
4242
43+
# Timeout configurations from agent server config
44+
proxy_connect_timeout {{.readiness_timeout}};
45+
proxy_send_timeout {{.download_timeout}};
46+
proxy_read_timeout {{.download_timeout}};
47+
4348
location ~ ^/(health|readiness)$ {
4449
proxy_pass http://agent-server;
50+
51+
# Use shorter timeout for health checks
52+
proxy_read_timeout {{.readiness_timeout}};
53+
proxy_send_timeout {{.readiness_timeout}};
54+
}
55+
56+
# Container runtime operations (preload/pull) need longer timeouts
57+
location ~ ^/preload/ {
58+
proxy_pass http://agent-server;
59+
60+
# Use container runtime timeout for these operations
61+
proxy_read_timeout {{.container_runtime_timeout}};
62+
proxy_send_timeout {{.container_runtime_timeout}};
63+
}
64+
65+
# Download operations
66+
location ~ ^/namespace/.*/blobs/ {
67+
proxy_pass http://agent-server;
68+
69+
# Use download timeout for blob operations
70+
proxy_read_timeout {{.download_timeout}};
71+
proxy_send_timeout {{.download_timeout}};
4572
}
4673
4774
location / {
4875
proxy_pass http://registry-backend;
4976
proxy_next_upstream error timeout http_404 http_500;
77+
78+
# Standard timeouts for registry operations
79+
proxy_read_timeout {{.download_timeout}};
80+
proxy_send_timeout {{.download_timeout}};
5081
}
5182
}
5283
`

‎nginx/config/origin.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,53 @@ server {
2828
gzip on;
2929
gzip_types text/plain test/csv application/json;
3030
31+
# Timeout configurations from origin server config
32+
proxy_connect_timeout {{.backend_timeout}};
33+
proxy_send_timeout {{.upload_timeout}};
34+
proxy_read_timeout {{.download_timeout}};
35+
36+
# Keepalive settings
37+
proxy_buffering off;
38+
proxy_request_buffering off;
39+
3140
location / {
3241
proxy_pass http://{{.server}};
42+
43+
# Pass original client info
44+
proxy_set_header Host $host;
45+
proxy_set_header X-Real-IP $remote_addr;
46+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
47+
proxy_set_header X-Forwarded-Proto $scheme;
48+
}
49+
50+
# Special handling for upload operations with longer timeout
51+
location ~ ^/namespace/.*/blobs/.*/uploads {
52+
proxy_pass http://{{.server}};
53+
54+
# Use upload timeout for these operations
55+
proxy_read_timeout {{.upload_timeout}};
56+
proxy_send_timeout {{.upload_timeout}};
57+
58+
# Pass original client info
59+
proxy_set_header Host $host;
60+
proxy_set_header X-Real-IP $remote_addr;
61+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
62+
proxy_set_header X-Forwarded-Proto $scheme;
63+
}
64+
65+
# Replication operations with their own timeout
66+
location ~ ^/namespace/.*/blobs/.*/remote {
67+
proxy_pass http://{{.server}};
68+
69+
# Use replication timeout for these operations
70+
proxy_read_timeout {{.replication_timeout}};
71+
proxy_send_timeout {{.replication_timeout}};
72+
73+
# Pass original client info
74+
proxy_set_header Host $host;
75+
proxy_set_header X-Real-IP $remote_addr;
76+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
77+
proxy_set_header X-Forwarded-Proto $scheme;
3378
}
3479
}
3580
`

‎nginx/config/tracker.go‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,36 @@ server {
2929
access_log {{.access_log_path}};
3030
error_log {{.error_log_path}};
3131
32+
# Timeout configurations from tracker server config
33+
proxy_connect_timeout {{.readiness_timeout}};
34+
proxy_send_timeout {{.announce_timeout}};
35+
proxy_read_timeout {{.announce_timeout}};
36+
3237
location / {
3338
proxy_pass http://tracker;
39+
40+
# Pass original client info
41+
proxy_set_header Host $host;
42+
proxy_set_header X-Real-IP $remote_addr;
43+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
44+
proxy_set_header X-Forwarded-Proto $scheme;
45+
}
46+
47+
# Health and readiness checks with shorter timeout
48+
location ~ ^/(health|readiness)$ {
49+
proxy_pass http://tracker;
50+
51+
proxy_read_timeout {{.readiness_timeout}};
52+
proxy_send_timeout {{.readiness_timeout}};
53+
54+
# Pass original client info
55+
proxy_set_header Host $host;
56+
proxy_set_header X-Real-IP $remote_addr;
57+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
58+
proxy_set_header X-Forwarded-Proto $scheme;
3459
}
3560
61+
# Metainfo requests need longer timeout (cached)
3662
location ~* ^/namespace/.*/blobs/.*/metainfo$ {
3763
proxy_pass http://tracker;
3864
@@ -41,6 +67,31 @@ server {
4167
proxy_cache_valid 200 5m;
4268
proxy_cache_valid any 1s;
4369
proxy_cache_lock on;
70+
71+
# Use metainfo timeout for these operations
72+
proxy_read_timeout {{.metainfo_timeout}};
73+
proxy_send_timeout {{.metainfo_timeout}};
74+
75+
# Pass original client info
76+
proxy_set_header Host $host;
77+
proxy_set_header X-Real-IP $remote_addr;
78+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
79+
proxy_set_header X-Forwarded-Proto $scheme;
80+
}
81+
82+
# Announce operations
83+
location ~ ^/announce {
84+
proxy_pass http://tracker;
85+
86+
# Use announce timeout for these operations
87+
proxy_read_timeout {{.announce_timeout}};
88+
proxy_send_timeout {{.announce_timeout}};
89+
90+
# Pass original client info
91+
proxy_set_header Host $host;
92+
proxy_set_header X-Real-IP $remote_addr;
93+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
94+
proxy_set_header X-Forwarded-Proto $scheme;
4495
}
4596
}
4697
`

‎nginx/nginx.go‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
// you may not use this file except in compliance with the License.
55
// You may obtain a copy of the License at
66
//
7-
// http://www.apache.org/licenses/LICENSE-2.0
7+
// http://www.apache.org/licenses/LICENSE-2.0
88
//
99
// Unless required by applicable law or agreed to in writing, software
1010
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,6 +23,7 @@ import (
2323
"path"
2424
"path/filepath"
2525
"text/template"
26+
"time"
2627

2728
"github.com/uber/kraken/nginx/config"
2829
"github.com/uber/kraken/utils/httputil"
@@ -249,3 +250,21 @@ func GetServer(net, addr string) string {
249250
}
250251
return addr
251252
}
253+
254+
func FormatDurationForNginx(d time.Duration) string {
255+
// Add 30s buffer to ensure Go server times out first for observability
256+
bufferedDuration := d + (30 * time.Second)
257+
258+
if bufferedDuration >= time.Minute {
259+
minutes := int(bufferedDuration.Minutes())
260+
if bufferedDuration == time.Duration(minutes)*time.Minute {
261+
return fmt.Sprintf("%dm", minutes)
262+
}
263+
}
264+
if bufferedDuration >= time.Second {
265+
seconds := int(bufferedDuration.Seconds())
266+
return fmt.Sprintf("%ds", seconds)
267+
}
268+
// Fallback to milliseconds for very short durations
269+
return fmt.Sprintf("%dms", bufferedDuration.Milliseconds())
270+
}

‎origin/cmd/cmd.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -410,8 +410,12 @@ func startServices(config Config, flags *Flags, server *blobserver.Server, sched
410410
log.Fatal(nginx.Run(
411411
config.Nginx,
412412
map[string]interface{}{
413-
"port": flags.BlobServerPort,
414-
"server": nginx.GetServer(config.BlobServer.Listener.Net, config.BlobServer.Listener.Addr),
413+
"port": flags.BlobServerPort,
414+
"server": nginx.GetServer(config.BlobServer.Listener.Net, config.BlobServer.Listener.Addr),
415+
"download_timeout": nginx.FormatDurationForNginx(config.BlobServer.DownloadTimeout),
416+
"upload_timeout": nginx.FormatDurationForNginx(config.BlobServer.UploadTimeout),
417+
"backend_timeout": nginx.FormatDurationForNginx(config.BlobServer.BackendTimeout),
418+
"replication_timeout": nginx.FormatDurationForNginx(config.BlobServer.ReplicationTimeout),
415419
},
416420
nginx.WithTLS(config.TLS)))
417421
}

‎tracker/cmd/cmd.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,11 @@ func Run(flags *Flags, opts ...Option) {
158158
log.Fatal(nginx.Run(config.Nginx, map[string]interface{}{
159159
"port": flags.Port,
160160
"server": nginx.GetServer(
161-
config.TrackerServer.Listener.Net, config.TrackerServer.Listener.Addr)},
161+
config.TrackerServer.Listener.Net, config.TrackerServer.Listener.Addr),
162+
// Pass timeout parameters from tracker server config
163+
"metainfo_timeout": nginx.FormatDurationForNginx(config.TrackerServer.MetaInfoTimeout),
164+
"announce_timeout": nginx.FormatDurationForNginx(config.TrackerServer.AnnounceTimeout),
165+
"readiness_timeout": nginx.FormatDurationForNginx(config.TrackerServer.ReadinessTimeout),
166+
},
162167
nginx.WithTLS(config.TLS)))
163168
}

0 commit comments

Comments
 (0)