Skip to content

Commit 0ae65f3

Browse files
drpcstream: install per-stream windows from an internal flow-control option
Streams install their send and receive windows from a FlowControl option carried in the internal (drpcopts) stream options. Keeping enablement internal-only makes accidental activation impossible for module consumers: bumping the drpc dependency mid-implementation cannot turn flow control on, because no public API reaches it. A later change promotes the option to the public Options once the version-gated enablement in CockroachDB is ready. Construction validates the configuration and panics on one that cannot make progress (silently disabling would drop the protection instead): all sizes positive, SplitSize bounded (negative means unsplit frames, so no bound holds; zero uses SplitData's 64 KiB default), and GrantThreshold + frame size <= StreamWindow -- up to GrantThreshold-1 bytes stay withheld at quiescence, and the remaining window must still cover the sender's next frame. Co-Authored-By: roachdev-claude <roachdev-claude-bot@cockroachlabs.com>
1 parent 9c97b8e commit 0ae65f3

3 files changed

Lines changed: 178 additions & 1 deletion

File tree

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright (C) 2026 Cockroach Labs.
2+
// See LICENSE for copying information.
3+
4+
package drpcstream
5+
6+
import (
7+
"context"
8+
"math"
9+
"testing"
10+
"time"
11+
12+
"github.com/zeebo/assert"
13+
14+
"storj.io/drpc/drpcwire"
15+
"storj.io/drpc/internal/drpcopts"
16+
)
17+
18+
// fcOptions returns stream Options with flow control enabled via the internal
19+
// option, the only way to enable it until it is promoted to a public option.
20+
func fcOptions(window, highWater, threshold int64) Options {
21+
opts := Options{SplitSize: 64 << 10}
22+
drpcopts.SetStreamFlowControl(&opts.Internal, drpcopts.FlowControl{
23+
Enabled: true,
24+
StreamWindow: window,
25+
HighWater: highWater,
26+
GrantThreshold: threshold,
27+
})
28+
return opts
29+
}
30+
31+
// Enabling flow control installs the send and receive windows, seeding the
32+
// send window with the configured stream window as initial credit.
33+
func TestStream_FlowControlEnabledInstallsWindows(t *testing.T) {
34+
mw := testMuxWriter(t)
35+
st := NewWithOptions(context.Background(), 1, mw, NewBufferPool(), fcOptions(256<<10, 4<<20, 128<<10))
36+
assert.That(t, st.sendw != nil)
37+
assert.That(t, st.recvw != nil)
38+
assert.Equal(t, st.sendw.available(), int64(256<<10))
39+
}
40+
41+
// Without flow control enabled, no windows are installed and behavior is
42+
// unchanged (ungated).
43+
func TestStream_FlowControlDisabledNoWindows(t *testing.T) {
44+
mw := testMuxWriter(t)
45+
st := NewWithOptions(context.Background(), 1, mw, NewBufferPool(), Options{SplitSize: 64 << 10})
46+
assert.That(t, st.sendw == nil)
47+
assert.That(t, st.recvw == nil)
48+
}
49+
50+
// An invalid flow-control configuration panics at construction rather than
51+
// deadlocking later or silently disabling the protection.
52+
func TestStream_FlowControlValidation(t *testing.T) {
53+
mustPanic := func(name string, opts Options) {
54+
t.Helper()
55+
defer func() {
56+
if recover() == nil {
57+
t.Fatalf("%s: expected construction to panic", name)
58+
}
59+
}()
60+
NewWithOptions(context.Background(), 1, testMuxWriter(t), NewBufferPool(), opts)
61+
}
62+
63+
mustPanic("zero window", fcOptions(0, 4<<20, 128<<10))
64+
mustPanic("zero high water", fcOptions(256<<10, 0, 128<<10))
65+
mustPanic("zero threshold", fcOptions(256<<10, 4<<20, 0))
66+
mustPanic("negative window", fcOptions(-1, 4<<20, 128<<10))
67+
68+
// Threshold + frame must fit in the window, or a quiescent receiver can
69+
// strand the sender below the next frame's cost.
70+
mustPanic("threshold+frame exceeds window", fcOptions(128<<10, 4<<20, 128<<10))
71+
72+
// A near-max threshold must not sneak past via int64 overflow of the
73+
// threshold+frame sum.
74+
mustPanic("near-max threshold overflow", fcOptions(256<<10, 4<<20, math.MaxInt64))
75+
76+
// SplitSize < 0 means unsplit (unbounded) frames: no bound holds.
77+
unbounded := fcOptions(256<<10, 4<<20, 64<<10)
78+
unbounded.SplitSize = -1
79+
mustPanic("unbounded SplitSize", unbounded)
80+
81+
// SplitSize 0 uses SplitData's 64 KiB default as the frame size.
82+
def := fcOptions(128<<10, 4<<20, 64<<10)
83+
def.SplitSize = 0
84+
NewWithOptions(context.Background(), 1, testMuxWriter(t), NewBufferPool(), def) // 64K+64K <= 128K: ok
85+
def2 := fcOptions(96<<10, 4<<20, 64<<10)
86+
def2.SplitSize = 0
87+
mustPanic("default frame exceeds window headroom", def2)
88+
}
89+
90+
// End to end via the option: an option-installed window gates a data write
91+
// that exceeds the initial credit, and an incoming grant resumes it. SplitSize
92+
// 2 with a 4-byte window means "hello" (frames 2+2+1) parks on its last frame.
93+
func TestStream_FlowControlOptionGatesAndResumes(t *testing.T) {
94+
mw := testMuxWriter(t)
95+
opts := Options{SplitSize: 2}
96+
drpcopts.SetStreamFlowControl(&opts.Internal, drpcopts.FlowControl{
97+
Enabled: true, StreamWindow: 4, HighWater: 1 << 20, GrantThreshold: 2,
98+
})
99+
st := NewWithOptions(context.Background(), 1, mw, NewBufferPool(), opts)
100+
101+
done := make(chan error, 1)
102+
go func() { done <- st.RawWrite(drpcwire.KindMessage, []byte("hello")) }() // 5 bytes > 4
103+
104+
select {
105+
case <-done:
106+
t.Fatal("write returned before sufficient credit")
107+
case <-time.After(blockShort):
108+
}
109+
110+
// A grant from the peer tops up the send window and resumes the write.
111+
assert.NoError(t, st.HandleFrame(drpcwire.WindowUpdateFrame(st.ID(), 1)))
112+
113+
select {
114+
case err := <-done:
115+
assert.NoError(t, err)
116+
case <-time.After(time.Second):
117+
t.Fatal("write did not resume after grant")
118+
}
119+
}

drpcstream/stream.go

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,33 @@ type Options struct {
3434
// All KindMessage data is compressed on send and decompressed on receive.
3535
Compression drpc.Compression
3636

37-
// Internal contains options that are for internal use only.
37+
// Internal contains options that are for internal use only. Per-stream
38+
// flow control is configured here (drpcopts.SetStreamFlowControl) and is
39+
// internal-only until the version-gated enablement work is ready.
3840
Internal drpcopts.Stream
3941
}
4042

43+
// validateFlowControl panics on a flow-control configuration that cannot make
44+
// progress: an invalid window is a programmer error, and disabling silently
45+
// would silently drop the protection. The frame size is SplitSize, or
46+
// SplitData's 64 KiB default when SplitSize is 0.
47+
func validateFlowControl(fc drpcopts.FlowControl, splitSize int) {
48+
frame := int64(splitSize)
49+
if splitSize == 0 {
50+
frame = 64 << 10
51+
}
52+
switch {
53+
case splitSize < 0:
54+
panic("drpcstream: flow control requires a bounded SplitSize")
55+
case fc.StreamWindow <= 0 || fc.HighWater <= 0 || fc.GrantThreshold <= 0:
56+
panic(fmt.Sprintf("drpcstream: flow-control sizes must be positive: %+v", fc))
57+
case frame > fc.StreamWindow || fc.GrantThreshold > fc.StreamWindow-frame:
58+
panic(fmt.Sprintf(
59+
"drpcstream: GrantThreshold (%d) + frame size (%d) exceeds StreamWindow (%d)",
60+
fc.GrantThreshold, frame, fc.StreamWindow))
61+
}
62+
}
63+
4164
// Stream represents an rpc actively happening on a transport.
4265
type Stream struct {
4366
ctx streamCtx
@@ -132,6 +155,15 @@ func NewWithOptions(
132155
onDequeue: drpcopts.GetStreamOnReceiveQueueDequeue(&opts.Internal),
133156
})
134157

158+
// When flow control is enabled, install the per-stream windows. The send
159+
// window starts with StreamWindow credit (statically agreed with the peer);
160+
// the receive window drives grant emission.
161+
if fc := drpcopts.GetStreamFlowControl(&opts.Internal); fc.Enabled {
162+
validateFlowControl(fc, opts.SplitSize)
163+
s.sendw = newSendWindow(fc.StreamWindow)
164+
s.recvw = newRecvWindow(fc.HighWater, fc.GrantThreshold)
165+
}
166+
135167
return s
136168
}
137169

internal/drpcopts/stream.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,34 @@ type Stream struct {
1717
stats *drpcstats.Stats
1818
onReceiveQueueEnqueue func(int64)
1919
onReceiveQueueDequeue func(int64)
20+
flowControl FlowControl
2021
}
2122

23+
// FlowControl configures per-stream flow control. Internal-only until the
24+
// CockroachDB version-gated enablement is ready; a later change promotes it to
25+
// a public option. The installation site validates it (see drpcstream).
26+
type FlowControl struct {
27+
// Enabled turns per-stream flow control on.
28+
Enabled bool
29+
30+
// StreamWindow is the sender's initial and nominal per-stream credit.
31+
StreamWindow int64
32+
33+
// HighWater is the receive-side buffered-byte mark at or above which
34+
// grants are withheld.
35+
HighWater int64
36+
37+
// GrantThreshold is the credit that must accrue before a grant is emitted,
38+
// coalescing many frames into one KindWindowUpdate.
39+
GrantThreshold int64
40+
}
41+
42+
// GetStreamFlowControl returns the FlowControl stored in the options.
43+
func GetStreamFlowControl(opts *Stream) FlowControl { return opts.flowControl }
44+
45+
// SetStreamFlowControl sets the FlowControl stored in the options.
46+
func SetStreamFlowControl(opts *Stream, fc FlowControl) { opts.flowControl = fc }
47+
2248
// GetStreamTransport returns the drpc.Transport stored in the options.
2349
func GetStreamTransport(opts *Stream) drpc.Transport { return opts.transport }
2450

0 commit comments

Comments
 (0)