Skip to content

Commit 7e6f6fe

Browse files
authored
feat(pool): add EachCtx for cancellable fan-out (#352)
Each has no way to observe cancellation, so a fan-out scanner (full port sweep, directory brute-force) keeps draining its whole queue even after ctrl-c or -max-time fires. EachCtx takes a context and stops feeding workers once it's cancelled, while still handing the ctx to the callback so an in-flight item can bail out too. Each is now a thin wrapper over EachCtx with context.Background(), so existing callers are unaffected until they opt in.
1 parent 75597fc commit 7e6f6fe

2 files changed

Lines changed: 71 additions & 4 deletions

File tree

internal/pool/pool.go

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,26 @@
1616
// holding it, the rest keep draining the queue instead of idling behind it.
1717
package pool
1818

19-
import "sync"
19+
import (
20+
"context"
21+
"sync"
22+
)
2023

2124
// Each runs fn for every item in items, concurrently, across at most workers
2225
// goroutines. order isn't preserved - fn must be safe to call from multiple
2326
// goroutines and guard any shared state itself. blocks until every item is done.
2427
func Each[T any](items []T, workers int, fn func(T)) {
28+
EachCtx(context.Background(), items, workers, func(_ context.Context, item T) {
29+
fn(item)
30+
})
31+
}
32+
33+
// EachCtx is Each with a context: once ctx is cancelled, workers stop pulling
34+
// new items off the queue (fn still gets called with ctx for the item it's
35+
// already holding, so it can bail out mid-item too). this is what lets a
36+
// fan-out scanner - a full port sweep, a directory brute-force - stop
37+
// promptly on ctrl-c / -max-time instead of draining the whole queue first.
38+
func EachCtx[T any](ctx context.Context, items []T, workers int, fn func(context.Context, T)) {
2539
if len(items) == 0 {
2640
return
2741
}
@@ -46,10 +60,14 @@ func Each[T any](items []T, workers int, fn func(T)) {
4660
for i := 0; i < workers; i++ {
4761
go func() {
4862
defer wg.Done()
49-
// pull until the queue is drained; a worker that finishes its
50-
// current item just grabs the next, which is the work-stealing.
63+
// pull until the queue is drained or ctx is cancelled; a worker
64+
// that finishes its current item just grabs the next, which is
65+
// the work-stealing.
5166
for item := range queue {
52-
fn(item)
67+
if ctx.Err() != nil {
68+
return
69+
}
70+
fn(ctx, item)
5371
}
5472
}()
5573
}

internal/pool/pool_test.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313
package pool
1414

1515
import (
16+
"context"
1617
"sync"
1718
"sync/atomic"
1819
"testing"
20+
"time"
1921
)
2022

2123
// every item runs exactly once across a spread of sizes and worker counts,
@@ -143,3 +145,50 @@ func TestEachCapsAtItemCount(t *testing.T) {
143145
t.Fatalf("peak concurrency %d exceeded item count %d", got, items)
144146
}
145147
}
148+
149+
// TestEachCtxStopsEarlyOnCancel proves the cancellation contract documented on
150+
// EachCtx (pool.go), for the -max-time / ctrl-c fan-out case.
151+
func TestEachCtxStopsEarlyOnCancel(t *testing.T) {
152+
const (
153+
items = 1000
154+
workers = 4
155+
)
156+
work := make([]int, items)
157+
158+
ctx, cancel := context.WithCancel(context.Background())
159+
160+
var processed int64
161+
var wg sync.WaitGroup
162+
wg.Add(1)
163+
go func() {
164+
defer wg.Done()
165+
EachCtx(ctx, work, workers, func(_ context.Context, _ int) {
166+
// cancel partway through so most items are still queued when it fires.
167+
if atomic.AddInt64(&processed, 1) == 20 {
168+
cancel()
169+
}
170+
// give the other workers a chance to observe the cancellation
171+
// before grabbing their next item, instead of racing straight
172+
// through the buffered channel.
173+
time.Sleep(time.Millisecond)
174+
})
175+
}()
176+
177+
done := make(chan struct{})
178+
go func() {
179+
wg.Wait()
180+
close(done)
181+
}()
182+
183+
select {
184+
case <-done:
185+
case <-time.After(5 * time.Second):
186+
t.Fatal("EachCtx did not return after cancellation; workers kept draining the queue")
187+
}
188+
189+
got := atomic.LoadInt64(&processed)
190+
if got >= items {
191+
t.Fatalf("EachCtx processed all %d items despite cancellation (processed=%d): cancel had no effect", items, got)
192+
}
193+
t.Logf("EachCtx stopped after %d/%d items once ctx was cancelled", got, items)
194+
}

0 commit comments

Comments
 (0)