Skip to content

Commit efc77b4

Browse files
drpcstream: add per-stream send-window credit primitive
Add sendWindow, the per-stream flow-control credit balance on the sender. acquire spends credit, blocking until enough is available; grant adds credit; close terminates the window. acquire returns early on close (with the close error) or context cancellation (with ctx.Err()), consuming no credit in either case; both are checked before any debit, so a canceled context never consumes credit or lets a frame proceed even when credit is available. Grants are no-revoke: grant takes an unsigned delta (the wire type), so a grant can only ever raise the balance, and the addition saturates at math.MaxInt64 so a large or malicious delta can never wrap it negative. The balance is a signed int64; acquire never drives it below zero, but the enablement layer may debit credit for bytes sent before flow control begins enforcing, leaving it negative, in which case acquire blocks until grants restore it. This is the primitive only -- it is not yet wired into the stream write path (that gate comes in a later commit). Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>
1 parent 4273044 commit efc77b4

2 files changed

Lines changed: 264 additions & 0 deletions

File tree

drpcstream/send_window.go

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Copyright (C) 2026 Cockroach Labs.
2+
// See LICENSE for copying information.
3+
4+
package drpcstream
5+
6+
import (
7+
"context"
8+
"math"
9+
"sync"
10+
)
11+
12+
// sendWindow is a per-stream flow-control credit balance on the sender. It
13+
// tracks how many more bytes the stream is allowed to put on the wire right
14+
// now. acquire spends credit (blocking until enough is available), grant adds
15+
// credit, and close terminates the window.
16+
//
17+
// Grants are no-revoke: grant is strictly additive, and the receiver never
18+
// takes back credit it has already issued. The balance is a signed int64;
19+
// acquire never drives it below zero, but the enablement layer may debit credit
20+
// for bytes sent before flow control begins enforcing, leaving it negative, in
21+
// which case acquire blocks until grants restore it.
22+
type sendWindow struct {
23+
mu sync.Mutex
24+
avail int64 // available credit; signed (may be negative — see doc)
25+
closed bool // set once by close; no further acquires succeed
26+
err error // terminal error returned by acquire after close
27+
notify chan struct{} // closed+replaced to wake parked acquirers
28+
}
29+
30+
// newSendWindow returns a sendWindow seeded with initial credit.
31+
func newSendWindow(initial int64) *sendWindow {
32+
return &sendWindow{avail: initial, notify: make(chan struct{})}
33+
}
34+
35+
// available returns the current credit balance.
36+
func (w *sendWindow) available() int64 {
37+
w.mu.Lock()
38+
defer w.mu.Unlock()
39+
return w.avail
40+
}
41+
42+
// acquire blocks until n bytes of credit are available, then debits them and
43+
// returns nil. It returns early if the window is closed (with the close error)
44+
// or ctx is canceled (with ctx.Err()), consuming no credit in either case. Both
45+
// are checked before any debit, so a canceled context never consumes credit or
46+
// lets a frame proceed even when credit is available.
47+
func (w *sendWindow) acquire(ctx context.Context, n int64) error {
48+
for {
49+
w.mu.Lock()
50+
switch {
51+
case w.closed:
52+
err := w.err
53+
w.mu.Unlock()
54+
return err
55+
case ctx.Err() != nil:
56+
w.mu.Unlock()
57+
return ctx.Err()
58+
case w.avail >= n:
59+
w.avail -= n
60+
w.mu.Unlock()
61+
return nil
62+
}
63+
// Snapshot the notify channel under the lock before parking, so a grant
64+
// or close that fires the instant we unlock is not missed.
65+
ch := w.notify
66+
w.mu.Unlock()
67+
68+
select {
69+
case <-ch:
70+
// Credit was granted or the window closed; loop and re-check.
71+
case <-ctx.Done():
72+
return ctx.Err()
73+
}
74+
}
75+
}
76+
77+
// grant adds n bytes of credit and wakes any parked acquirer. n is unsigned
78+
// (the wire delta type), so a grant can only ever raise the balance — no-revoke.
79+
// The addition saturates at math.MaxInt64 so a large or malicious delta can
80+
// never wrap the balance negative. Grants after close are ignored (the window
81+
// is dead).
82+
func (w *sendWindow) grant(n uint64) {
83+
w.mu.Lock()
84+
if !w.closed {
85+
if n >= uint64(math.MaxInt64) {
86+
w.avail = math.MaxInt64
87+
} else if sum := w.avail + int64(n); sum < w.avail {
88+
w.avail = math.MaxInt64 // overflowed past MaxInt64
89+
} else {
90+
w.avail = sum
91+
}
92+
w.wakeLocked()
93+
}
94+
w.mu.Unlock()
95+
}
96+
97+
// close terminates the window with err, waking every parked acquirer, which
98+
// then returns err. Subsequent acquires also return err. It is a no-op if the
99+
// window is already closed.
100+
func (w *sendWindow) close(err error) {
101+
w.mu.Lock()
102+
if !w.closed {
103+
w.closed = true
104+
w.err = err
105+
w.wakeLocked()
106+
}
107+
w.mu.Unlock()
108+
}
109+
110+
// wakeLocked broadcasts to all parked acquirers by closing the current notify
111+
// channel and installing a fresh one. It must be called with w.mu held.
112+
func (w *sendWindow) wakeLocked() {
113+
close(w.notify)
114+
w.notify = make(chan struct{})
115+
}

drpcstream/send_window_test.go

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
// Copyright (C) 2026 Cockroach Labs.
2+
// See LICENSE for copying information.
3+
4+
package drpcstream
5+
6+
import (
7+
"context"
8+
"errors"
9+
"math"
10+
"testing"
11+
"time"
12+
13+
"github.com/zeebo/assert"
14+
"github.com/zeebo/errs"
15+
)
16+
17+
// blockShort is how long we wait to conclude that an acquire is (correctly)
18+
// blocked before we release it.
19+
const blockShort = 20 * time.Millisecond
20+
21+
func TestSendWindowAcquireImmediate(t *testing.T) {
22+
w := newSendWindow(1000)
23+
assert.Equal(t, w.available(), int64(1000))
24+
25+
assert.NoError(t, w.acquire(context.Background(), 400))
26+
assert.Equal(t, w.available(), int64(600))
27+
28+
assert.NoError(t, w.acquire(context.Background(), 600))
29+
assert.Equal(t, w.available(), int64(0))
30+
}
31+
32+
func TestSendWindowGrantsAccumulate(t *testing.T) {
33+
w := newSendWindow(0)
34+
w.grant(100)
35+
w.grant(50)
36+
assert.Equal(t, w.available(), int64(150))
37+
38+
assert.NoError(t, w.acquire(context.Background(), 150))
39+
assert.Equal(t, w.available(), int64(0))
40+
}
41+
42+
func TestSendWindowAcquireBlocksUntilGrant(t *testing.T) {
43+
w := newSendWindow(100)
44+
done := make(chan error, 1)
45+
go func() { done <- w.acquire(context.Background(), 300) }()
46+
47+
// Not enough credit yet: acquire must block.
48+
select {
49+
case <-done:
50+
t.Fatal("acquire returned before sufficient credit")
51+
case <-time.After(blockShort):
52+
}
53+
54+
w.grant(250) // 100 + 250 = 350 >= 300
55+
56+
select {
57+
case err := <-done:
58+
assert.NoError(t, err)
59+
case <-time.After(time.Second):
60+
t.Fatal("acquire did not return after grant")
61+
}
62+
assert.Equal(t, w.available(), int64(50))
63+
}
64+
65+
func TestSendWindowAcquireContextCancel(t *testing.T) {
66+
w := newSendWindow(0)
67+
ctx, cancel := context.WithCancel(context.Background())
68+
done := make(chan error, 1)
69+
go func() { done <- w.acquire(ctx, 100) }()
70+
71+
select {
72+
case <-done:
73+
t.Fatal("acquire returned before cancellation")
74+
case <-time.After(blockShort):
75+
}
76+
77+
cancel()
78+
79+
select {
80+
case err := <-done:
81+
assert.That(t, errors.Is(err, context.Canceled))
82+
case <-time.After(time.Second):
83+
t.Fatal("acquire did not wake on context cancellation")
84+
}
85+
// Credit was not consumed by a failed acquire.
86+
assert.Equal(t, w.available(), int64(0))
87+
}
88+
89+
func TestSendWindowGrantSaturates(t *testing.T) {
90+
// Adding to a near-max balance saturates at MaxInt64 instead of wrapping.
91+
w := newSendWindow(math.MaxInt64 - 10)
92+
w.grant(100)
93+
assert.Equal(t, w.available(), int64(math.MaxInt64))
94+
95+
// A single delta larger than MaxInt64 (the wire delta is uint64) saturates
96+
// rather than becoming a negative balance.
97+
w2 := newSendWindow(0)
98+
w2.grant(math.MaxUint64)
99+
assert.Equal(t, w2.available(), int64(math.MaxInt64))
100+
assert.That(t, w2.available() > 0) // never wrapped negative
101+
102+
// A grant that raises a negative balance is applied exactly (no saturation).
103+
w3 := newSendWindow(0)
104+
w3.avail = -300
105+
w3.grant(500)
106+
assert.Equal(t, w3.available(), int64(200))
107+
}
108+
109+
func TestSendWindowAcquireCanceledCtxWithCredit(t *testing.T) {
110+
w := newSendWindow(1000)
111+
ctx, cancel := context.WithCancel(context.Background())
112+
cancel() // already canceled, with credit available
113+
114+
err := w.acquire(ctx, 100)
115+
assert.That(t, errors.Is(err, context.Canceled))
116+
// A canceled context must not consume credit even though it was available.
117+
assert.Equal(t, w.available(), int64(1000))
118+
}
119+
120+
func TestSendWindowCloseWakesAcquire(t *testing.T) {
121+
w := newSendWindow(0)
122+
closeErr := errs.New("terminated")
123+
done := make(chan error, 1)
124+
go func() { done <- w.acquire(context.Background(), 100) }()
125+
126+
select {
127+
case <-done:
128+
t.Fatal("acquire returned before close")
129+
case <-time.After(blockShort):
130+
}
131+
132+
w.close(closeErr)
133+
134+
select {
135+
case err := <-done:
136+
assert.That(t, errors.Is(err, closeErr))
137+
case <-time.After(time.Second):
138+
t.Fatal("acquire did not wake on close")
139+
}
140+
}
141+
142+
func TestSendWindowAcquireAfterClose(t *testing.T) {
143+
w := newSendWindow(1000)
144+
closeErr := errs.New("closed")
145+
w.close(closeErr)
146+
147+
// Even though credit is available, a closed window returns the close error.
148+
assert.That(t, errors.Is(w.acquire(context.Background(), 1), closeErr))
149+
}

0 commit comments

Comments
 (0)