Skip to content

Commit 4d08d0b

Browse files
drpcstream: add per-stream send-window credit primitive
Add sendWindow, the per-stream flow-control credit balance on the sender. acquire spends credit, blocking until enough is available; grant adds credit; close terminates the window. acquire returns early on close (with the close error) or context cancellation (with ctx.Err()), consuming no credit in either case; both are checked before any debit, so a canceled context never consumes credit or lets a frame proceed even when credit is available. Grants are no-revoke: grant takes an unsigned delta (the wire type), so a grant can only ever raise the balance, and the addition saturates at math.MaxInt64 so a large or malicious delta can never wrap it negative. The balance is a signed int64; acquire never drives it below zero, but the enablement layer may debit credit for bytes sent before flow control begins enforcing, leaving it negative, in which case acquire blocks until grants restore it. This is the primitive only -- it is not yet wired into the stream write path (that gate comes in a later commit). Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>
1 parent 5c858e3 commit 4d08d0b

2 files changed

Lines changed: 324 additions & 0 deletions

File tree

drpcstream/send_window.go

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
// Copyright (C) 2026 Cockroach Labs.
2+
// See LICENSE for copying information.
3+
4+
package drpcstream
5+
6+
import (
7+
"context"
8+
"io"
9+
"math"
10+
"sync"
11+
)
12+
13+
// sendWindow is a per-stream flow-control credit balance on the sender. It
14+
// tracks how many more bytes the stream is allowed to put on the wire right
15+
// now. acquire spends credit (blocking until enough is available), grant adds
16+
// credit, and close terminates the window.
17+
//
18+
// avail is a signed int64;
19+
// acquire never drives it below zero, but the enablement layer may debit credit
20+
// for bytes sent before flow control begins enforcing, leaving it negative, in
21+
// which case acquire blocks until grants restore it.
22+
type sendWindow struct {
23+
mu sync.Mutex
24+
avail int64 // available credit; signed (maybe negative during enablement)
25+
closed bool // set once by close; no further acquires succeed
26+
err error // terminal error returned by acquire after close
27+
notify chan struct{} // closed+replaced to wake parked acquirers
28+
}
29+
30+
// newSendWindow returns a sendWindow seeded with initial credit.
31+
func newSendWindow(initial int64) *sendWindow {
32+
return &sendWindow{avail: initial, notify: make(chan struct{})}
33+
}
34+
35+
// available returns the current credit balance.
36+
func (w *sendWindow) available() int64 {
37+
w.mu.Lock()
38+
defer w.mu.Unlock()
39+
return w.avail
40+
}
41+
42+
// acquire debits n bytes of credit, blocking until available, and returns nil.
43+
// It returns early (consuming no credit) if the window is closed or ctx is
44+
// canceled
45+
func (w *sendWindow) acquire(ctx context.Context, n int64) error {
46+
for {
47+
w.mu.Lock()
48+
switch {
49+
case w.closed:
50+
err := w.err
51+
w.mu.Unlock()
52+
return err
53+
case ctx.Err() != nil:
54+
w.mu.Unlock()
55+
return ctx.Err()
56+
case n <= 0:
57+
// Nothing to acquire; after the terminal cases (so a closed/canceled
58+
// window still fails) and never debits (so a negative n cannot add credit).
59+
w.mu.Unlock()
60+
return nil
61+
case w.avail >= n:
62+
w.avail -= n
63+
w.mu.Unlock()
64+
return nil
65+
}
66+
// Snapshot the notify channel under the lock before parking, so a grant
67+
// or close that fires the instant we unlock is not missed.
68+
ch := w.notify
69+
w.mu.Unlock()
70+
71+
select {
72+
case <-ch:
73+
// Credit was granted or the window closed; loop and re-check.
74+
case <-ctx.Done():
75+
return ctx.Err()
76+
}
77+
}
78+
}
79+
80+
// grant raises the balance by n and wakes any parked acquirer. n is unsigned,
81+
// so a grant never lowers the balance. Grants after close are ignored.
82+
func (w *sendWindow) grant(n uint64) {
83+
w.mu.Lock()
84+
if !w.closed {
85+
w.avail = applyGrant(w.avail, n)
86+
w.wakeLocked()
87+
}
88+
w.mu.Unlock()
89+
}
90+
91+
// applyGrant returns avail + n with an upper bound of math.MaxInt64.
92+
// n is the wire delta (unsigned), so the result never drops below avail.
93+
func applyGrant(avail int64, n uint64) int64 {
94+
if avail >= 0 {
95+
if n > uint64(math.MaxInt64-avail) {
96+
return math.MaxInt64
97+
}
98+
return avail + int64(n)
99+
}
100+
// A negative avail (before FC enablement) is repaid before any positive
101+
// credit starts accruing.
102+
deficit := uint64(-avail) // |avail| as uint64; -math.MinInt64 wraps to its magnitude
103+
if n <= deficit {
104+
return -int64(deficit - n) // debt only partly repaid; result still <= 0
105+
}
106+
if rem := n - deficit; rem <= uint64(math.MaxInt64) {
107+
return int64(rem)
108+
}
109+
return math.MaxInt64
110+
}
111+
112+
// close terminates the window with err, waking every parked acquirer, which
113+
// then returns err. Subsequent acquires also return err. It is a no-op if the
114+
// window is already closed.
115+
func (w *sendWindow) close(err error) {
116+
if err == nil {
117+
err = io.EOF // never nil: acquire must not report success on a closed window
118+
}
119+
w.mu.Lock()
120+
if !w.closed {
121+
w.closed = true
122+
w.err = err
123+
w.wakeLocked()
124+
}
125+
w.mu.Unlock()
126+
}
127+
128+
// wakeLocked broadcasts to all parked acquirers by closing the current notify
129+
// channel and installing a fresh one. It must be called with w.mu held.
130+
func (w *sendWindow) wakeLocked() {
131+
close(w.notify)
132+
w.notify = make(chan struct{})
133+
}

drpcstream/send_window_test.go

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Copyright (C) 2026 Cockroach Labs.
2+
// See LICENSE for copying information.
3+
4+
package drpcstream
5+
6+
import (
7+
"context"
8+
"errors"
9+
"io"
10+
"math"
11+
"testing"
12+
"time"
13+
14+
"github.com/zeebo/assert"
15+
"github.com/zeebo/errs"
16+
)
17+
18+
// blockShort is how long we wait to conclude that an acquire is (correctly)
19+
// blocked before we release it.
20+
const blockShort = 20 * time.Millisecond
21+
22+
func TestSendWindowAcquireImmediate(t *testing.T) {
23+
w := newSendWindow(1000)
24+
assert.Equal(t, w.available(), int64(1000))
25+
26+
assert.NoError(t, w.acquire(context.Background(), 400))
27+
assert.Equal(t, w.available(), int64(600))
28+
29+
assert.NoError(t, w.acquire(context.Background(), 600))
30+
assert.Equal(t, w.available(), int64(0))
31+
}
32+
33+
func TestSendWindowGrantsAccumulate(t *testing.T) {
34+
w := newSendWindow(0)
35+
w.grant(100)
36+
w.grant(50)
37+
assert.Equal(t, w.available(), int64(150))
38+
39+
assert.NoError(t, w.acquire(context.Background(), 150))
40+
assert.Equal(t, w.available(), int64(0))
41+
}
42+
43+
func TestSendWindowAcquireBlocksUntilGrant(t *testing.T) {
44+
w := newSendWindow(100)
45+
done := make(chan error, 1)
46+
go func() { done <- w.acquire(context.Background(), 300) }()
47+
48+
// Not enough credit yet: acquire must block.
49+
select {
50+
case <-done:
51+
t.Fatal("acquire returned before sufficient credit")
52+
case <-time.After(blockShort):
53+
}
54+
55+
w.grant(250) // 100 + 250 = 350 >= 300
56+
57+
select {
58+
case err := <-done:
59+
assert.NoError(t, err)
60+
case <-time.After(time.Second):
61+
t.Fatal("acquire did not return after grant")
62+
}
63+
assert.Equal(t, w.available(), int64(50))
64+
}
65+
66+
func TestSendWindowAcquireContextCancel(t *testing.T) {
67+
w := newSendWindow(0)
68+
ctx, cancel := context.WithCancel(context.Background())
69+
done := make(chan error, 1)
70+
go func() { done <- w.acquire(ctx, 100) }()
71+
72+
select {
73+
case <-done:
74+
t.Fatal("acquire returned before cancellation")
75+
case <-time.After(blockShort):
76+
}
77+
78+
cancel()
79+
80+
select {
81+
case err := <-done:
82+
assert.That(t, errors.Is(err, context.Canceled))
83+
case <-time.After(time.Second):
84+
t.Fatal("acquire did not wake on context cancellation")
85+
}
86+
// Credit was not consumed by a failed acquire.
87+
assert.Equal(t, w.available(), int64(0))
88+
}
89+
90+
func TestSendWindowGrantSaturates(t *testing.T) {
91+
// Adding to a near-max balance saturates at MaxInt64 instead of wrapping.
92+
w := newSendWindow(math.MaxInt64 - 10)
93+
w.grant(100)
94+
assert.Equal(t, w.available(), int64(math.MaxInt64))
95+
96+
// A single delta larger than MaxInt64 (the wire delta is uint64) saturates
97+
// rather than becoming a negative balance.
98+
w2 := newSendWindow(0)
99+
w2.grant(math.MaxUint64)
100+
assert.Equal(t, w2.available(), int64(math.MaxInt64))
101+
assert.That(t, w2.available() > 0) // never wrapped negative
102+
103+
// A grant that raises a negative balance is applied exactly (no saturation).
104+
w3 := newSendWindow(0)
105+
w3.avail = -300
106+
w3.grant(500)
107+
assert.Equal(t, w3.available(), int64(200))
108+
109+
// A very large grant against a negative balance repays the debt before
110+
// clamping: the true sum -300 + MaxInt64 fits, so it must not saturate to
111+
// MaxInt64 (which would erase the 300 of pre-enforcement debt).
112+
w4 := newSendWindow(0)
113+
w4.avail = -300
114+
w4.grant(math.MaxInt64)
115+
assert.Equal(t, w4.available(), int64(math.MaxInt64-300))
116+
117+
// Only when the true sum truly overflows does it clamp.
118+
w5 := newSendWindow(0)
119+
w5.avail = -300
120+
w5.grant(math.MaxUint64)
121+
assert.Equal(t, w5.available(), int64(math.MaxInt64))
122+
}
123+
124+
func TestSendWindowAcquireCanceledCtxWithCredit(t *testing.T) {
125+
w := newSendWindow(1000)
126+
ctx, cancel := context.WithCancel(context.Background())
127+
cancel() // already canceled, with credit available
128+
129+
err := w.acquire(ctx, 100)
130+
assert.That(t, errors.Is(err, context.Canceled))
131+
// A canceled context must not consume credit even though it was available.
132+
assert.Equal(t, w.available(), int64(1000))
133+
}
134+
135+
func TestSendWindowCloseWakesAcquire(t *testing.T) {
136+
w := newSendWindow(0)
137+
closeErr := errs.New("terminated")
138+
done := make(chan error, 1)
139+
go func() { done <- w.acquire(context.Background(), 100) }()
140+
141+
select {
142+
case <-done:
143+
t.Fatal("acquire returned before close")
144+
case <-time.After(blockShort):
145+
}
146+
147+
w.close(closeErr)
148+
149+
select {
150+
case err := <-done:
151+
assert.That(t, errors.Is(err, closeErr))
152+
case <-time.After(time.Second):
153+
t.Fatal("acquire did not wake on close")
154+
}
155+
}
156+
157+
func TestSendWindowAcquireAfterClose(t *testing.T) {
158+
w := newSendWindow(1000)
159+
closeErr := errs.New("closed")
160+
w.close(closeErr)
161+
162+
// Even though credit is available, a closed window returns the close error.
163+
assert.That(t, errors.Is(w.acquire(context.Background(), 1), closeErr))
164+
}
165+
166+
func TestSendWindowCloseNilError(t *testing.T) {
167+
w := newSendWindow(1000)
168+
w.close(nil) // closing with nil must not let a later acquire report success
169+
assert.That(t, errors.Is(w.acquire(context.Background(), 1), io.EOF))
170+
}
171+
172+
func TestSendWindowAcquireNonPositive(t *testing.T) {
173+
w := newSendWindow(100)
174+
assert.NoError(t, w.acquire(context.Background(), 0))
175+
assert.NoError(t, w.acquire(context.Background(), -5))
176+
// Non-positive acquire consumes nothing; a negative one must not add credit.
177+
assert.Equal(t, w.available(), int64(100))
178+
}
179+
180+
func TestSendWindowAcquireZeroObservesTerminalState(t *testing.T) {
181+
// A zero-length acquire (empty frame) must still observe a canceled context...
182+
ctx, cancel := context.WithCancel(context.Background())
183+
cancel()
184+
assert.That(t, errors.Is(newSendWindow(100).acquire(ctx, 0), context.Canceled))
185+
186+
// ...and a closed window, rather than reporting success.
187+
w := newSendWindow(100)
188+
closeErr := errs.New("terminated")
189+
w.close(closeErr)
190+
assert.That(t, errors.Is(w.acquire(context.Background(), 0), closeErr))
191+
}

0 commit comments

Comments
 (0)