Skip to content

Commit e631fbc

Browse files
committed
ian/adding_concurrency_limit
1 parent f6d01ef commit e631fbc

7 files changed

Lines changed: 133 additions & 26 deletions

File tree

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ require (
99
github.com/docker/docker v28.5.2+incompatible
1010
github.com/gorilla/websocket v1.5.3
1111
github.com/rs/zerolog v1.31.0
12+
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
13+
gopkg.in/yaml.v3 v3.0.1
1214
)
1315

1416
require (
@@ -53,6 +55,5 @@ require (
5355
golang.org/x/sys v0.39.0 // indirect
5456
golang.org/x/time v0.14.0 // indirect
5557
google.golang.org/protobuf v1.36.10 // indirect
56-
gopkg.in/yaml.v3 v3.0.1 // indirect
5758
gotest.tools/v3 v3.5.2 // indirect
5859
)

go.sum

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,12 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
108108
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
109109
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
110110
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
111+
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
112+
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
111113
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
112114
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
115+
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
116+
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
113117
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
114118
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
115119
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -164,6 +168,8 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
164168
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
165169
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
166170
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
171+
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
172+
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
167173
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
168174
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
169175
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
@@ -215,6 +221,7 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr
215221
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
216222
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
217223
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
224+
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
218225
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
219226
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
220227
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -261,6 +268,8 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
261268
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
262269
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
263270
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
271+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
272+
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
264273
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
265274
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
266275
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/config/config.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ import (
1111

1212
// FileConfig represents the top-level YAML configuration file.
1313
type FileConfig struct {
14-
WorkerID string `yaml:"worker_id"`
15-
Cleanup *bool `yaml:"cleanup"`
16-
Backend BackendConfig `yaml:"backend"`
14+
WorkerID string `yaml:"worker_id"`
15+
Cleanup *bool `yaml:"cleanup"`
16+
MaxConcurrentTasks *int `yaml:"max_concurrent_tasks"`
17+
Backend BackendConfig `yaml:"backend"`
1718
}
1819

1920
// BackendConfig contains the backend selection. At most one backend may be configured.

internal/config/config_test.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,3 +256,35 @@ func TestLoadFileNotFound(t *testing.T) {
256256
t.Fatal("expected error for missing file")
257257
}
258258
}
259+
260+
func TestLoadMaxConcurrentTasks(t *testing.T) {
261+
t.Run("parses max_concurrent_tasks when set", func(t *testing.T) {
262+
path := writeTestConfig(t, `
263+
worker_id: "test"
264+
max_concurrent_tasks: 5
265+
`)
266+
cfg, err := Load(path)
267+
if err != nil {
268+
t.Fatalf("unexpected error: %v", err)
269+
}
270+
if cfg.MaxConcurrentTasks == nil {
271+
t.Fatal("expected max_concurrent_tasks to be set")
272+
}
273+
if *cfg.MaxConcurrentTasks != 5 {
274+
t.Errorf("max_concurrent_tasks = %d, want 5", *cfg.MaxConcurrentTasks)
275+
}
276+
})
277+
278+
t.Run("max_concurrent_tasks is nil when not set", func(t *testing.T) {
279+
path := writeTestConfig(t, `
280+
worker_id: "test"
281+
`)
282+
cfg, err := Load(path)
283+
if err != nil {
284+
t.Fatalf("unexpected error: %v", err)
285+
}
286+
if cfg.MaxConcurrentTasks != nil {
287+
t.Errorf("expected max_concurrent_tasks to be nil, got %d", *cfg.MaxConcurrentTasks)
288+
}
289+
})
290+
}

internal/types/messages.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const (
1212
MessageTypeTaskAssignment MessageType = "task_assignment"
1313
MessageTypeTaskClaimed MessageType = "task_claimed"
1414
MessageTypeTaskFailed MessageType = "task_failed"
15+
MessageTypeTaskRejected MessageType = "task_rejected"
1516
MessageTypeHeartbeat MessageType = "heartbeat"
1617
)
1718

@@ -53,6 +54,13 @@ type TaskFailedMessage struct {
5354
Message string `json:"message"`
5455
}
5556

57+
// TaskRejectedMessage is sent from worker to server when the worker cannot accept the task
58+
// (e.g. at maximum concurrency). The server should keep the task queued rather than marking it failed.
59+
type TaskRejectedMessage struct {
60+
TaskID string `json:"task_id"`
61+
Reason string `json:"reason"`
62+
}
63+
5664
type TaskDefinition struct {
5765
Prompt string `json:"prompt"`
5866
}

internal/worker/worker.go

Lines changed: 54 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/warpdotdev/oz-agent-worker/internal/common"
1313
"github.com/warpdotdev/oz-agent-worker/internal/log"
1414
"github.com/warpdotdev/oz-agent-worker/internal/types"
15+
"golang.org/x/sync/semaphore"
1516
)
1617

1718
const (
@@ -25,12 +26,13 @@ const (
2526
)
2627

2728
type Config struct {
28-
APIKey string
29-
WorkerID string
30-
WebSocketURL string
31-
ServerRootURL string
32-
LogLevel string
33-
BackendType string // "docker" or "direct"
29+
APIKey string
30+
WorkerID string
31+
WebSocketURL string
32+
ServerRootURL string
33+
LogLevel string
34+
BackendType string // "docker" or "direct"
35+
MaxConcurrentTasks int // 0 means unlimited
3436

3537
// Backend-specific configs. Only the one matching BackendType should be set.
3638
Docker *DockerBackendConfig
@@ -49,6 +51,7 @@ type Worker struct {
4951
activeTasks map[string]context.CancelFunc
5052
tasksMutex sync.Mutex
5153
backend Backend
54+
taskSemaphore *semaphore.Weighted // nil when unlimited
5255
}
5356

5457
func New(ctx context.Context, config Config) (*Worker, error) {
@@ -79,6 +82,11 @@ func New(ctx context.Context, config Config) (*Worker, error) {
7982
return nil, err
8083
}
8184

85+
var taskSemaphore *semaphore.Weighted
86+
if config.MaxConcurrentTasks > 0 {
87+
taskSemaphore = semaphore.NewWeighted(int64(config.MaxConcurrentTasks))
88+
}
89+
8290
return &Worker{
8391
config: config,
8492
ctx: workerCtx,
@@ -87,6 +95,7 @@ func New(ctx context.Context, config Config) (*Worker, error) {
8795
sendChan: make(chan []byte, 256),
8896
activeTasks: make(map[string]context.CancelFunc),
8997
backend: backend,
98+
taskSemaphore: taskSemaphore,
9099
}, nil
91100
}
92101

@@ -298,6 +307,17 @@ func (w *Worker) handleMessage(message []byte) {
298307
func (w *Worker) handleTaskAssignment(assignment *types.TaskAssignmentMessage) {
299308
log.Infof(w.ctx, "Received task assignment: taskID=%s, title=%s", assignment.TaskID, assignment.Task.Title)
300309

310+
// Check concurrency limit before claiming the task.
311+
if w.taskSemaphore != nil {
312+
if !w.taskSemaphore.TryAcquire(1) {
313+
log.Warnf(w.ctx, "Rejecting task %s: worker at maximum concurrency (%d)", assignment.TaskID, w.config.MaxConcurrentTasks)
314+
if err := w.sendTaskRejected(assignment.TaskID, "worker at maximum concurrency"); err != nil {
315+
log.Errorf(w.ctx, "Failed to send task rejected message: %v", err)
316+
}
317+
return
318+
}
319+
}
320+
301321
// It's important to update the task state to claimed as the task lifecycle treats this as a dependency to advance to further states.
302322
if err := w.sendTaskClaimed(assignment.TaskID); err != nil {
303323
log.Errorf(w.ctx, "Failed to send task claimed message: %v", err)
@@ -369,6 +389,10 @@ func (w *Worker) executeTask(ctx context.Context, assignment *types.TaskAssignme
369389
w.tasksMutex.Lock()
370390
delete(w.activeTasks, assignment.TaskID)
371391
w.tasksMutex.Unlock()
392+
393+
if w.taskSemaphore != nil {
394+
w.taskSemaphore.Release(1)
395+
}
372396
}()
373397

374398
taskID := assignment.TaskID
@@ -410,6 +434,30 @@ func (w *Worker) sendTaskClaimed(taskID string) error {
410434
return w.sendMessage(msgBytes)
411435
}
412436

437+
func (w *Worker) sendTaskRejected(taskID, reason string) error {
438+
rejectedMsg := types.TaskRejectedMessage{
439+
TaskID: taskID,
440+
Reason: reason,
441+
}
442+
443+
data, err := json.Marshal(rejectedMsg)
444+
if err != nil {
445+
return fmt.Errorf("failed to marshal task rejected message: %w", err)
446+
}
447+
448+
msg := types.WebSocketMessage{
449+
Type: types.MessageTypeTaskRejected,
450+
Data: data,
451+
}
452+
453+
msgBytes, err := json.Marshal(msg)
454+
if err != nil {
455+
return fmt.Errorf("failed to marshal websocket message: %w", err)
456+
}
457+
458+
return w.sendMessage(msgBytes)
459+
}
460+
413461
func (w *Worker) sendTaskFailed(taskID, message string) error {
414462
failedMsg := types.TaskFailedMessage{
415463
TaskID: taskID,

main.go

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,17 @@ import (
1515
)
1616

1717
var CLI struct {
18-
ConfigFile string `help:"Path to YAML config file" type:"path"`
19-
Backend string `help:"Backend type (docker or direct)" enum:"docker,direct," default:""`
20-
APIKey string `help:"API key for authentication" env:"WARP_API_KEY" required:""`
21-
WorkerID string `help:"Worker host identifier (required via flag or config file)"`
22-
WebSocketURL string `default:"wss://oz.warp.dev/api/v1/selfhosted/worker/ws" hidden:""`
23-
ServerRootURL string `default:"https://app.warp.dev" hidden:""`
24-
LogLevel string `help:"Log level (debug, info, warn, error)" default:"info" enum:"debug,info,warn,error"`
25-
NoCleanup bool `help:"Do not remove containers after execution (for debugging)"`
26-
Volumes []string `help:"Volume mounts for task containers (format: HOST_PATH:CONTAINER_PATH or HOST_PATH:CONTAINER_PATH:MODE)" short:"v"`
27-
Env []string `help:"Environment variables for task containers (format: KEY=VALUE or KEY to pass through from host)" short:"e"`
18+
ConfigFile string `help:"Path to YAML config file" type:"path"`
19+
Backend string `help:"Backend type (docker or direct)" enum:"docker,direct," default:""`
20+
APIKey string `help:"API key for authentication" env:"WARP_API_KEY" required:""`
21+
WorkerID string `help:"Worker host identifier (required via flag or config file)"`
22+
WebSocketURL string `default:"wss://oz.warp.dev/api/v1/selfhosted/worker/ws" hidden:""`
23+
ServerRootURL string `default:"https://app.warp.dev" hidden:""`
24+
LogLevel string `help:"Log level (debug, info, warn, error)" default:"info" enum:"debug,info,warn,error"`
25+
NoCleanup bool `help:"Do not remove containers after execution (for debugging)"`
26+
Volumes []string `help:"Volume mounts for task containers (format: HOST_PATH:CONTAINER_PATH or HOST_PATH:CONTAINER_PATH:MODE)" short:"v"`
27+
Env []string `help:"Environment variables for task containers (format: KEY=VALUE or KEY to pass through from host)" short:"e"`
28+
MaxConcurrentTasks int `help:"Maximum number of tasks to run concurrently (0 for unlimited)" default:"0"`
2829
}
2930

3031
func main() {
@@ -116,13 +117,20 @@ func mergeConfig(fileConfig *config.FileConfig) (worker.Config, error) {
116117
return worker.Config{}, err
117118
}
118119

120+
// Resolve max_concurrent_tasks: CLI (non-zero) > config file > 0 (unlimited).
121+
maxConcurrentTasks := CLI.MaxConcurrentTasks
122+
if maxConcurrentTasks == 0 && fileConfig != nil && fileConfig.MaxConcurrentTasks != nil {
123+
maxConcurrentTasks = *fileConfig.MaxConcurrentTasks
124+
}
125+
119126
wc := worker.Config{
120-
APIKey: CLI.APIKey,
121-
WorkerID: workerID,
122-
WebSocketURL: CLI.WebSocketURL,
123-
ServerRootURL: CLI.ServerRootURL,
124-
LogLevel: CLI.LogLevel,
125-
BackendType: backendType,
127+
APIKey: CLI.APIKey,
128+
WorkerID: workerID,
129+
WebSocketURL: CLI.WebSocketURL,
130+
ServerRootURL: CLI.ServerRootURL,
131+
LogLevel: CLI.LogLevel,
132+
BackendType: backendType,
133+
MaxConcurrentTasks: maxConcurrentTasks,
126134
}
127135

128136
switch backendType {

0 commit comments

Comments
 (0)