diff --git a/src/TurboHTTP.Tests/Protocol/Body/BodyDrainSlotSpec.cs b/src/TurboHTTP.Tests/Protocol/Body/BodyDrainSlotSpec.cs index b71e7b5b5..c632fc59e 100644 --- a/src/TurboHTTP.Tests/Protocol/Body/BodyDrainSlotSpec.cs +++ b/src/TurboHTTP.Tests/Protocol/Body/BodyDrainSlotSpec.cs @@ -13,11 +13,10 @@ public void Initialize_should_set_identity_fields() var cts = new CancellationTokenSource(); using var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token); - slot.Initialize(42, stream, contentLength: 3, CancellationToken.None, linked); + slot.Initialize(42, stream, CancellationToken.None, linked); Assert.Equal(42, slot.StreamId); Assert.Same(stream, slot.BodyStream); - Assert.Equal(3L, slot.ContentLength); Assert.Same(linked, slot.LinkedCts); } @@ -32,66 +31,16 @@ public void BeginRead_should_set_IsReadInFlight() } [Fact(Timeout = 5000)] - public void CompleteSyncRead_should_clear_IsReadInFlight() + public void CompleteRead_should_clear_IsReadInFlight() { var slot = new BodyDrainSlot(); slot.BeginRead(); - slot.CompleteSyncRead(); + slot.CompleteRead(); Assert.False(slot.IsReadInFlight); } - [Fact(Timeout = 5000)] - public void CompleteAsyncRead_should_clear_IsReadInFlight_and_reset_sync_counter() - { - var slot = new BodyDrainSlot(); - slot.BeginRead(); - slot.IncrementSyncReads(100); - slot.IncrementSyncReads(100); - - slot.CompleteAsyncRead(); - - Assert.False(slot.IsReadInFlight); - Assert.Equal(0, slot.ConsecutiveSyncReads); - } - - [Fact(Timeout = 5000)] - public void IncrementSyncReads_should_return_false_below_threshold() - { - var slot = new BodyDrainSlot(); - - var result = slot.IncrementSyncReads(3); - - Assert.False(result); - Assert.Equal(1, slot.ConsecutiveSyncReads); - } - - [Fact(Timeout = 5000)] - public void IncrementSyncReads_should_return_true_at_threshold() - { - var slot = new BodyDrainSlot(); - - slot.IncrementSyncReads(3); - slot.IncrementSyncReads(3); - var result = slot.IncrementSyncReads(3); - - Assert.True(result); - Assert.Equal(3, slot.ConsecutiveSyncReads); - } - - [Fact(Timeout = 5000)] - public void ResetSyncReads_should_zero_counter() - { - var slot = new BodyDrainSlot(); - slot.IncrementSyncReads(100); - slot.IncrementSyncReads(100); - - slot.ResetSyncReads(); - - Assert.Equal(0, slot.ConsecutiveSyncReads); - } - [Fact(Timeout = 5000)] public void MarkOrphaned_should_set_IsOrphaned() { @@ -103,49 +52,18 @@ public void MarkOrphaned_should_set_IsOrphaned() } [Fact(Timeout = 5000)] - public void MarkDrainComplete_should_set_IsDrainComplete() - { - var slot = new BodyDrainSlot(); - - slot.MarkDrainComplete(); - - Assert.True(slot.IsDrainComplete); - } - - [Fact(Timeout = 5000)] - public void StoreLimbo_should_set_HasLimbo_and_LimboData() - { - var slot = new BodyDrainSlot(); - var data = new byte[] { 10, 20, 30 }; - - slot.StoreLimbo(data); - - Assert.True(slot.HasLimbo); - Assert.Equal(data, slot.LimboData.ToArray()); - } - - [Fact(Timeout = 5000)] - public void ShrinkLimbo_should_advance_slice() + public void ReservedWindow_should_default_to_zero() { var slot = new BodyDrainSlot(); - var data = new byte[] { 1, 2, 3, 4, 5 }; - slot.StoreLimbo(data); - - slot.ShrinkLimbo(2); - - Assert.Equal([3, 4, 5], slot.LimboData.ToArray()); + Assert.Equal(0, slot.ReservedWindow); } [Fact(Timeout = 5000)] - public void ClearLimbo_should_clear_HasLimbo_and_LimboData() + public void ReservedWindow_should_persist_across_reads() { var slot = new BodyDrainSlot(); - slot.StoreLimbo(new byte[] { 1, 2 }); - - slot.ClearLimbo(); - - Assert.False(slot.HasLimbo); - Assert.True(slot.LimboData.IsEmpty); + slot.ReservedWindow = 8 * 1024; + Assert.Equal(8 * 1024, slot.ReservedWindow); } [Fact(Timeout = 5000)] @@ -194,7 +112,7 @@ public void DisposeResources_should_dispose_buffer_and_LinkedCts() var stream = new MemoryStream([]); var cts = new CancellationTokenSource(); using var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token); - slot.Initialize(1, stream, null, CancellationToken.None, linked); + slot.Initialize(1, stream, CancellationToken.None, linked); slot.EnsureBuffer(256); slot.DisposeResources(); @@ -210,30 +128,24 @@ public void Reset_should_clear_all_state() var stream = new MemoryStream([1, 2, 3]); var cts = new CancellationTokenSource(); using var linked = CancellationTokenSource.CreateLinkedTokenSource(cts.Token); - slot.Initialize(7, stream, 3, CancellationToken.None, linked); + slot.Initialize(7, stream, CancellationToken.None, linked); slot.EnsureBuffer(256); slot.BeginRead(); slot.MarkOrphaned(); - slot.StoreLimbo(new byte[] { 9 }); - slot.MarkDrainComplete(); - slot.IncrementSyncReads(100); + slot.ReservedWindow = 16 * 1024; slot.Reset(); Assert.Equal(0, slot.StreamId); Assert.Null(slot.BodyStream); - Assert.Null(slot.ContentLength); #pragma warning disable xUnit1051 // SUT behavior: asserts slot resets RequestCt to default, not test cooperative cancellation Assert.Equal(default, slot.RequestCt); #pragma warning restore xUnit1051 Assert.Null(slot.LinkedCts); Assert.Null(slot.Buffer); + Assert.Equal(0, slot.ReservedWindow); Assert.False(slot.IsReadInFlight); Assert.False(slot.IsOrphaned); - Assert.False(slot.HasLimbo); - Assert.True(slot.LimboData.IsEmpty); - Assert.False(slot.IsDrainComplete); - Assert.Equal(0, slot.ConsecutiveSyncReads); } [Fact(Timeout = 5000)] @@ -241,7 +153,7 @@ public void Reset_should_work_for_long_streamId() { var slot = new BodyDrainSlot(); var stream = new MemoryStream([]); - slot.Initialize(99L, stream, null, CancellationToken.None, null); + slot.Initialize(99L, stream, CancellationToken.None, null); slot.Reset(); diff --git a/src/TurboHTTP.Tests/Protocol/Body/BodyPumpBaseSpec.cs b/src/TurboHTTP.Tests/Protocol/Body/BodyPumpBaseSpec.cs new file mode 100644 index 000000000..7b598aadb --- /dev/null +++ b/src/TurboHTTP.Tests/Protocol/Body/BodyPumpBaseSpec.cs @@ -0,0 +1,816 @@ +using Akka.Actor; +using TurboHTTP.Pooling; +using TurboHTTP.Protocol.Body; + +namespace TurboHTTP.Tests.Protocol.Body; + +public sealed class BodyPumpBaseSpec +{ + private sealed class TestPump : BodyPumpBase + { + public TestPump(IBodyDrainTarget target, ConnectionPoolContext pool, CancellationTokenSource cts) + : base(target, pool, cts) { } + + public int Credits => GetCredits(); + public int Budget => GetBudget(); + public int ActiveStreamCount => GetActiveStreamCount(); + } + + private sealed class FakeTarget : IBodyDrainTarget + { + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand { get; set; } + public int PreferredChunkSize { get; set; } = 16 * 1024; + public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; + public List Completed { get; } = []; + public List<(int StreamId, Exception Reason)> Failed { get; } = []; + + public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) + => Emitted.Add((streamId, data.ToArray(), endStream)); + + public void OnDrainComplete(int streamId) => Completed.Add(streamId); + public void OnDrainFailed(int streamId, Exception reason) => Failed.Add((streamId, reason)); + } + + private static MemoryStream MakeBody(int size) + { + var data = new byte[size]; + for (var i = 0; i < size; i++) + { + data[i] = (byte)(i % 256); + } + + return new MemoryStream(data); + } + + [Fact(Timeout = 5000)] + public void AddCredit_should_accumulate_credits() + { + var target = new FakeTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + pump.AddCredit(); + pump.AddCredit(); + pump.AddCredit(); + + Assert.Equal(3, pump.Credits); + } + + [Fact(Timeout = 5000)] + public void AddCredit_should_cap_at_max_budget() + { + var target = new FakeTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + for (var i = 0; i < 100; i++) + { + pump.AddCredit(); + } + + Assert.Equal(48, pump.Credits); + } + + [Fact(Timeout = 5000)] + public void Register_should_read_immediately_when_HasPendingDemand() + { + var target = new FakeTarget { HasPendingDemand = true }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + + Assert.True(target.Emitted.Count >= 1); + } + + [Fact(Timeout = 5000)] + public void Register_should_not_read_without_demand_or_credits() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + + Assert.Empty(target.Emitted); + } + + [Fact(Timeout = 5000)] + public void AddCredit_should_trigger_read_round_when_threshold_reached() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + Assert.Empty(target.Emitted); + + // Threshold for 1 active stream = min(budget/2, 1*2) = 2 + pump.AddCredit(); + pump.AddCredit(); + + Assert.True(target.Emitted.Count >= 1); + } + + [Fact(Timeout = 5000)] + public void ReadRound_should_decrement_credits_per_read() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + + for (var i = 0; i < 10; i++) + { + pump.AddCredit(); + } + + Assert.True(pump.Credits < 10); + } + + [Fact(Timeout = 5000)] + public void CancelAll_should_clear_all_state() + { + var target = new FakeTarget { HasPendingDemand = false }; + var cts = new CancellationTokenSource(); + var pump = new TestPump(target, new ConnectionPoolContext(), cts); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + pump.CancelAll(); + + Assert.True(cts.IsCancellationRequested); + Assert.Equal(0, pump.ActiveStreamCount); + } + + [Fact(Timeout = 5000)] + public void Cancel_should_cleanup_idle_slot() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + pump.Cancel(0); + + Assert.Equal(0, pump.ActiveStreamCount); + } + + [Fact(Timeout = 5000)] + public void HandleReadComplete_should_emit_endStream_on_empty_body() + { + var target = new FakeTarget { HasPendingDemand = true }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = new MemoryStream([]); + + pump.Register(0, body, CancellationToken.None); + + Assert.Single(target.Emitted); + Assert.True(target.Emitted[0].EndStream); + Assert.Single(target.Completed); + } + + [Fact(Timeout = 5000)] + public void HandleReadFailed_should_call_OnDrainFailed() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + var error = new IOException("test error"); + + pump.Register(0, body, CancellationToken.None); + pump.HandleReadFailed(0, error); + + Assert.Single(target.Failed); + Assert.Equal(0, target.Failed[0].StreamId); + Assert.Same(error, target.Failed[0].Reason); + } + + [Fact(Timeout = 5000)] + public void Budget_should_be_initialized_within_valid_range() + { + var target = new FakeTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + Assert.InRange(pump.Budget, 8, 48); + } + + // Round-robin fairness + + [Fact(Timeout = 5000)] + public void ReadRound_should_serve_each_stream_before_second_turn_with_two_streams() + { + // Both streams registered with HasPendingDemand=false so we control when reads fire. + // We then add enough credits to trigger a read round and verify both stream IDs appear + // before either appears a second time. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + // Two bodies large enough to not complete in a single read (> 16 KB chunk) + var body0 = MakeBody(64 * 1024); + var body1 = MakeBody(64 * 1024); + + pump.Register(0, body0, CancellationToken.None); + pump.Register(1, body1, CancellationToken.None); + + // Add enough credits to exceed the read-round threshold (min(budget/2, 2*2) = 4). + for (var i = 0; i < 20; i++) + { + pump.AddCredit(); + } + + // Collect the stream IDs of the first two non-EOF emits. + var firstTwo = target.Emitted.Where(e => !e.EndStream).Take(2).Select(e => e.StreamId).ToList(); + Assert.Equal(2, firstTwo.Distinct().Count()); + } + + [Fact(Timeout = 5000)] + public void ReadRound_should_serve_all_three_streams_before_second_turn() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + var bodySize = 64 * 1024; + pump.Register(0, MakeBody(bodySize), CancellationToken.None); + pump.Register(1, MakeBody(bodySize), CancellationToken.None); + pump.Register(2, MakeBody(bodySize), CancellationToken.None); + + // Add credits well above threshold to trigger a multi-read round. + for (var i = 0; i < 48; i++) + { + pump.AddCredit(); + } + + // Each stream should have been served at least once. + var streamIds = target.Emitted.Where(e => !e.EndStream).Select(e => e.StreamId).ToList(); + Assert.Contains(0, streamIds); + Assert.Contains(1, streamIds); + Assert.Contains(2, streamIds); + + // Verify no stream gets two consecutive reads before the others each get one. + // The first three non-EOF reads must cover all three stream IDs. + var firstThree = streamIds.Take(3).ToList(); + Assert.Equal(3, firstThree.Distinct().Count()); + } + + // Adaptive budget + + [Fact(Timeout = 5000)] + public void Budget_should_clamp_at_MinBudget_8_under_slow_calls() + { + var target = new FakeTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + // Force the EMA to the slow-end by sleeping between AddCredit calls. + // 10 ms = SlowThresholdTicks, so we need intervals >= 10 ms. + // Use Thread.Sleep to space calls apart enough for the EMA to converge. + // 5 calls at 15 ms each should push EMA above SlowThresholdTicks (10 ms). + for (var i = 0; i < 5; i++) + { + Thread.Sleep(15); + pump.AddCredit(); + } + + // Budget should be at the minimum (8) because intervals are slow. + Assert.Equal(8, pump.Budget); + } + + [Fact(Timeout = 5000)] + public void Budget_should_clamp_at_MaxBudget_48_under_rapid_calls() + { + var target = new FakeTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + // Rapid calls: no sleep between them, so EMA should converge toward FastThresholdTicks (0.5 ms). + // After enough iterations the EMA should saturate to MaxBudget = 48. + for (var i = 0; i < 50; i++) + { + pump.AddCredit(); + } + + // After 50 rapid credits the budget should be at maximum. + Assert.Equal(48, pump.Budget); + } + + [Fact(Timeout = 5000)] + public void Budget_should_decrease_from_fast_to_slow_after_pause() + { + var target = new FakeTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + // Establish a fast budget first. + for (var i = 0; i < 20; i++) + { + pump.AddCredit(); + } + + var budgetAfterFast = pump.Budget; + + // Now pause for 20 ms (above SlowThresholdTicks = 10 ms) and then call AddCredit. + Thread.Sleep(20); + pump.AddCredit(); + + // Budget should have decreased (EMA shifted toward slow interval). + Assert.True(pump.Budget < budgetAfterFast || pump.Budget == 8, + $"Expected budget to decrease below {budgetAfterFast} after a slow interval."); + } + + // Completion-trigger + + [Fact(Timeout = 5000)] + public void HandleReadComplete_should_trigger_follow_up_read_when_credits_remain() + { + // Arrange: register a body large enough to require multiple reads. + // Force the pump to have credits > 0 before the first read completes. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(64 * 1024); + + pump.Register(0, body, CancellationToken.None); + + // With sync MemoryStream and credit reclaim, the first AddCredit that + // reaches threshold reads the entire body in one self-driving chain. + pump.AddCredit(); + + // Body should be fully drained (4 data chunks + 1 endStream) after a + // single credit triggers the self-driving sync read loop. + Assert.True(target.Emitted.Count >= 1, + "Sync read credit reclaim should drain the body with minimal credits."); + } + + [Fact(Timeout = 5000)] + public void Register_without_demand_should_read_after_sufficient_credits_added() + { + // With HasPendingDemand=false, a registered stream does not read immediately. + // Adding credits eventually accumulates to the threshold and fires a read round. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(64 * 1024); + + pump.Register(0, body, CancellationToken.None); + + // No credits yet: no reads. + Assert.Empty(target.Emitted); + + // Add enough credits to guarantee a read round fires (threshold is at most budget/2 = 4 when budget=8). + for (var i = 0; i < 10; i++) + { + pump.AddCredit(); + } + + // Reads should have fired. + Assert.NotEmpty(target.Emitted); + } + + // Cancellation — additional patterns + + [Fact(Timeout = 5000)] + public void Cancel_of_already_completed_stream_should_be_noop() + { + var target = new FakeTarget { HasPendingDemand = true }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = new MemoryStream([]); + + // Register empty body — completes immediately on Register (HasPendingDemand=true). + pump.Register(0, body, CancellationToken.None); + Assert.Single(target.Completed); + + // Cancelling after completion should not throw and should be a no-op. + pump.Cancel(0); + Assert.Equal(0, pump.ActiveStreamCount); + } + + [Fact(Timeout = 5000)] + public void CancelAll_should_zero_credits_and_active_streams() + { + var target = new FakeTarget { HasPendingDemand = false }; + var cts = new CancellationTokenSource(); + var pump = new TestPump(target, new ConnectionPoolContext(), cts); + + pump.Register(0, MakeBody(64 * 1024), CancellationToken.None); + pump.Register(1, MakeBody(64 * 1024), CancellationToken.None); + + pump.CancelAll(); + + Assert.Equal(0, pump.ActiveStreamCount); + Assert.Equal(0, pump.Credits); + Assert.True(cts.IsCancellationRequested); + } + + [Fact(Timeout = 5000)] + public void Cancel_multiple_streams_should_remove_only_targeted_stream() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + pump.Register(0, MakeBody(64 * 1024), CancellationToken.None); + pump.Register(1, MakeBody(64 * 1024), CancellationToken.None); + pump.Register(2, MakeBody(64 * 1024), CancellationToken.None); + + pump.Cancel(1); + + // Stream 1 removed; streams 0 and 2 still active. + Assert.Equal(2, pump.ActiveStreamCount); + } + + // Regression: Bug 1 — re-enqueue before emit (stale queueSize snapshot) + // Before the fix: ProcessReadResult re-enqueued AFTER EmitDataFrames. If EmitDataFrames + // triggered an inline AddCredit (simulating the feedback from OnOutboundFlushed), that + // AddCredit → TryReadNextEligible found an empty queue and did nothing. The pump stalled + // permanently because the stream was never re-enqueued. + // Fix: re-enqueue BEFORE EmitDataFrames so the inline AddCredit sees the stream. + + private sealed class InlineCreditFakeTarget : IBodyDrainTarget + { + private TestPump? _pump; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand { get; set; } + public int PreferredChunkSize { get; set; } = 16 * 1024; + public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; + public List Completed { get; } = []; + public List<(int StreamId, Exception Reason)> Failed { get; } = []; + + public void SetPump(TestPump pump) => _pump = pump; + + public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) + { + Emitted.Add((streamId, data.ToArray(), endStream)); + // Simulate inline feedback: an outbound flush immediately gives back a credit. + // Bug scenario: if the stream was re-enqueued AFTER this call, TryReadNextEligible + // called inside AddCredit would see an empty ready queue and skip the stream. + if (!endStream && _pump is not null) + { + _pump.AddCredit(); + } + } + + public void OnDrainComplete(int streamId) => Completed.Add(streamId); + public void OnDrainFailed(int streamId, Exception reason) => Failed.Add((streamId, reason)); + } + + [Fact(Timeout = 5000)] + public void ProcessReadResult_should_complete_drain_when_AddCredit_called_inline_from_EmitDataFrames() + { + // Arrange: a FakeTarget that calls pump.AddCredit() inside EmitDataFrames. + // The fix ensures the stream is re-enqueued BEFORE EmitDataFrames runs, so the + // inline AddCredit finds the stream in the ready queue and continues draining. + // Without the fix (re-enqueue AFTER EmitDataFrames), the inline AddCredit sees an + // empty queue and the body never completes. + var target = new InlineCreditFakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + target.SetPump(pump); + + // Body: 3 chunks of 16 KB → 3 data frames then 1 endStream frame. + var body = MakeBody(3 * 16 * 1024); + pump.Register(0, body, CancellationToken.None); + + // Seed exactly one credit — the inline AddCredit fired by each EmitDataFrames call + // continues the drain. The entire body must complete without external credits. + pump.AddCredit(); + + // The body should be fully drained: 3 data + 1 endStream + drain-complete. + Assert.Single(target.Completed); + Assert.Equal(0, target.Completed[0]); + } + + // Regression: Bug 2 — threshold too high for a single stream + // Before the fix: threshold = min(budget/2, activeStreams * 2). + // With 1 stream and budget ≈ 28: threshold = min(14, 2) = 2. + // Adding exactly 1 credit never reached the threshold → permanent stall. + // Fix: threshold = max(min(budget/2, activeStreams), 1) so threshold = 1 for 1 stream. + + [Fact(Timeout = 5000)] + public void TryStartReadRound_should_read_with_single_credit_when_only_one_stream_active() + { + // Arrange: single stream registered with no pending demand so reads only fire via credits. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(100); + + pump.Register(0, body, CancellationToken.None); + Assert.Empty(target.Emitted); + + // Act: add exactly 1 credit. + // Fix: threshold for 1 active stream is max(min(budget/2, 1), 1) = 1, so 1 credit is enough. + // Bug: threshold was min(budget/2, 1*2) = 2, so 1 credit < 2 → no read → permanent stall. + pump.AddCredit(); + + // Assert: the pump issued at least one read from a single credit. + Assert.NotEmpty(target.Emitted); + } + + // Regression: Bug 5 — sync credit starvation + // The pump must not stall when draining sync bodies. Each credit triggers one read; + // inline AddCredit from EmitDataFrames (on targets that support it) or OnOutboundFlushed + // replenishes credits. With sufficient credits, a sync body must drain completely. + + [Fact(Timeout = 5000)] + public void PerformRead_should_drain_body_with_sufficient_credits() + { + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(16 * 1024); + + pump.Register(0, body, CancellationToken.None); + Assert.Empty(target.Emitted); + + // Two credits: one for the data read (16KB), one for the EOF read (0 bytes). + pump.AddCredit(); + pump.AddCredit(); + + Assert.Contains(target.Emitted, e => !e.EndStream); + Assert.Single(target.Completed); + } + + // Regression: Bug 6 — double slot cleanup on drain complete + // Before the fix: on EOF, the order was EmitDataFrames(endStream:true) → OnDrainComplete → + // CloseStream → Cancel → CleanupSlot (first), then CleanupSlot again (second). + // Double pool return corrupts pool state (NullReferenceException on next rent). + // Fix: CleanupSlot BEFORE OnDrainComplete so Cancel finds no slot. + + private sealed class CancelOnDrainTarget : IBodyDrainTarget + { + private TestPump? _pump; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand { get; set; } = true; + public int PreferredChunkSize { get; set; } = 16 * 1024; + public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; + public List Completed { get; } = []; + public List<(int StreamId, Exception Reason)> Failed { get; } = []; + + public void SetPump(TestPump pump) => _pump = pump; + + public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) + => Emitted.Add((streamId, data.ToArray(), endStream)); + + public void OnDrainComplete(int streamId) + { + Completed.Add(streamId); + // Simulate CloseStream → _pump.Cancel(streamId) called inside OnDrainComplete. + // Bug: slot was still in _activeSlots at this point → Cancel would CleanupSlot again + // (double pool return). Fix: slot is cleaned up BEFORE OnDrainComplete fires, so + // Cancel here finds no slot and is a safe no-op. + _pump?.Cancel(streamId); + } + + public void OnDrainFailed(int streamId, Exception reason) => Failed.Add((streamId, reason)); + } + + [Fact(Timeout = 5000)] + public void ProcessReadResult_should_survive_Cancel_called_inside_OnDrainComplete() + { + // Arrange: a target that calls pump.Cancel(streamId) inside OnDrainComplete, simulating + // the CloseStream cascade. With the fix, the slot is removed before OnDrainComplete fires + // so Cancel is a safe no-op. Without the fix, Cancel would find the slot still registered + // and call CleanupSlot a second time → double pool return → corrupted state. + // + // An empty body produces EOF on the very first read, exercising the cleanup-before-notify + // path directly (bytesRead = 0 → CleanupSlot → OnDrainComplete). + var target = new CancelOnDrainTarget(); + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + target.SetPump(pump); + + // Empty body: HasPendingDemand=true → TryReadNextEligible fires → EOF immediately → + // CleanupSlot (fix: before notify) → OnDrainComplete → Cancel(0) → no-op. + pump.Register(0, new MemoryStream([]), CancellationToken.None); + + // Body drains to completion. The Cancel inside OnDrainComplete must be a safe no-op. + Assert.Single(target.Completed); + Assert.Equal(0, pump.ActiveStreamCount); + + // Verify pool integrity: register a second body — the slot must be reusable. + // Without the fix, the slot would have been double-returned and the pool is corrupted, + // causing a NullReferenceException or corrupted state here. + pump.Register(1, new MemoryStream([]), CancellationToken.None); + Assert.Equal(2, target.Completed.Count); + } + + // Edge case: stale DrainReadComplete after CancelAll + + [Fact(Timeout = 5000)] + public void HandleReadComplete_should_be_silently_ignored_after_CancelAll() + { + // Arrange: register a body so a slot exists, then CancelAll clears _activeSlots. + // A stale DrainReadComplete message (e.g. in-flight before CancelAll) arrives afterward. + // HandleReadComplete guards with TryGetValue and returns early when no slot is found. + var target = new FakeTarget { HasPendingDemand = false }; + var cts = new CancellationTokenSource(); + var pump = new TestPump(target, new ConnectionPoolContext(), cts); + + pump.Register(0, MakeBody(100), CancellationToken.None); + pump.CancelAll(); + + // Act: stale completion arrives for streamId 0 — must not throw. + var ex = Record.Exception(() => pump.HandleReadComplete(0, 50)); + Assert.Null(ex); + + // No data should have been emitted after CancelAll. + Assert.Empty(target.Emitted); + Assert.Empty(target.Completed); + } + + [Fact(Timeout = 5000)] + public void HandleReadFailed_should_be_silently_ignored_after_CancelAll() + { + // Arrange: same as above but for the failure path. + var target = new FakeTarget { HasPendingDemand = false }; + var cts = new CancellationTokenSource(); + var pump = new TestPump(target, new ConnectionPoolContext(), cts); + + pump.Register(0, MakeBody(100), CancellationToken.None); + pump.CancelAll(); + + // Act: stale failure arrives — must not throw or call OnDrainFailed. + var error = new IOException("stale error"); + var ex = Record.Exception(() => pump.HandleReadFailed(0, error)); + Assert.Null(ex); + + Assert.Empty(target.Failed); + } + + // Edge case: single-byte body + + [Fact(Timeout = 5000)] + public void Register_should_drain_single_byte_body_with_one_data_and_endStream() + { + // A 1-byte body must produce exactly 1 data emission (endStream=false) and + // 1 endStream emission (endStream=true, zero bytes), then OnDrainComplete. + // Two credits are needed: one for the data read, one for the EOF read. + // HasPendingDemand=false so we control when reads fire via AddCredit. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = new MemoryStream([42]); + + pump.Register(0, body, CancellationToken.None); + Assert.Empty(target.Emitted); + + // First credit → data read (1 byte). Second credit → EOF read (0 bytes). + pump.AddCredit(); + pump.AddCredit(); + + // Verify exactly: [data frame, endStream frame] + var dataFrame = Assert.Single(target.Emitted.Where(e => !e.EndStream).ToList()); + var eofFrame = Assert.Single(target.Emitted.Where(e => e.EndStream).ToList()); + var singleByte = Assert.Single(dataFrame.Data); + Assert.Equal(42, singleByte); + Assert.Empty(eofFrame.Data); + Assert.Single(target.Completed); + } + + // Edge case: body exactly equal to chunkSize + + [Fact(Timeout = 5000)] + public void Register_should_produce_exactly_one_data_and_one_endStream_when_body_equals_chunkSize() + { + // A body exactly chunkSize (16 KB) should emit 1 data frame (full chunk) + // followed by 1 endStream frame from the EOF read. + // Two credits: one for the data read, one for the EOF read. + var target = new FakeTarget { HasPendingDemand = false, PreferredChunkSize = 16 * 1024 }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = MakeBody(16 * 1024); + + pump.Register(0, body, CancellationToken.None); + Assert.Empty(target.Emitted); + + // Drive exactly 2 reads: data + EOF. + pump.AddCredit(); + pump.AddCredit(); + + // Exactly 1 data emit (non-endStream) + 1 endStream emit. + var dataEmits = target.Emitted.Where(e => !e.EndStream).ToList(); + var eofEmits = target.Emitted.Where(e => e.EndStream).ToList(); + Assert.Single(dataEmits); + Assert.Single(eofEmits); + Assert.Equal(16 * 1024, dataEmits[0].Data.Length); + Assert.Single(target.Completed); + } + + // Edge case: multiple cancellations leave queue clean + + [Fact(Timeout = 5000)] + public void Cancel_three_of_five_streams_should_leave_only_two_active_and_serve_only_them() + { + // Register 5 streams (no demand so nothing reads immediately), cancel 3. + // Adding credits must serve only the 2 remaining streams and not trip on cancelled ones. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + + for (var id = 0; id < 5; id++) + { + pump.Register(id, MakeBody(64 * 1024), CancellationToken.None); + } + + pump.Cancel(1); + pump.Cancel(3); + pump.Cancel(4); + + Assert.Equal(2, pump.ActiveStreamCount); + + // Drive reads with plenty of credits. + for (var i = 0; i < 48; i++) + { + pump.AddCredit(); + } + + // All emitted stream IDs must be from the non-cancelled set {0, 2}. + var emittedIds = target.Emitted.Select(e => e.StreamId).Distinct().ToHashSet(); + Assert.DoesNotContain(1, emittedIds); + Assert.DoesNotContain(3, emittedIds); + Assert.DoesNotContain(4, emittedIds); + } + + // Edge case: cancel during sync read chain + + private sealed class CancelMidDrainTarget : IBodyDrainTarget + { + private TestPump? _pump; + private int _emitCount; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand { get; set; } + public int PreferredChunkSize { get; set; } = 16 * 1024; + public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; + public List Completed { get; } = []; + public List<(int StreamId, Exception Reason)> Failed { get; } = []; + + public void SetPump(TestPump pump) => _pump = pump; + + public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) + { + Emitted.Add((streamId, data.ToArray(), endStream)); + // Cancel the stream on the first data emission to interrupt the drain chain. + if (!endStream && Interlocked.Increment(ref _emitCount) == 1) + { + _pump?.Cancel(streamId); + } + } + + public void OnDrainComplete(int streamId) => Completed.Add(streamId); + public void OnDrainFailed(int streamId, Exception reason) => Failed.Add((streamId, reason)); + } + + [Fact(Timeout = 5000)] + public void Cancel_during_sync_drain_should_stop_pump_cleanly() + { + // A large body (8 chunks) is registered with a target that cancels the stream + // after the first data emission. The pump must stop emitting after cancellation — + // no double-cleanup, no throws, no further emissions for that stream. + var target = new CancelMidDrainTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + target.SetPump(pump); + + var bodySize = 8 * 16 * 1024; + pump.Register(0, MakeBody(bodySize), CancellationToken.None); + + // Seed one credit to start the drain; the target cancels on first emit. + pump.AddCredit(); + + // The pump must have stopped: active stream count should be 0. + Assert.Equal(0, pump.ActiveStreamCount); + // No exception was thrown (implicit — if we reach here, it didn't throw). + // At most 2 emissions before cancel (1 data + possibly 1 re-enqueue attempt). + Assert.True(target.Emitted.Count <= 2, + $"Expected at most 2 emits before cancel stopped the drain, got {target.Emitted.Count}."); + } + + // Edge case: ReadAsync throws synchronously (not via faulted ValueTask) + + private sealed class SynchronousThrowStream : Stream + { + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + => throw new IOException("sync throw from ReadAsync"); + } + + [Fact(Timeout = 5000)] + public void ReadAsync_that_throws_synchronously_should_propagate_as_exception() + { + // StartRead calls ReadAsync inside PerformRead. + // If ReadAsync throws synchronously (not via a faulted ValueTask), the exception + // propagates out of PerformRead → DrainReady → AddCredit → caller. + // This test documents the current behavior: the pump does NOT catch synchronous throws + // from ReadAsync; the caller receives the exception directly. + var target = new FakeTarget { HasPendingDemand = false }; + var pump = new TestPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); + var body = new SynchronousThrowStream(); + + pump.Register(0, body, CancellationToken.None); + + // AddCredit triggers a read, which calls ReadAsync → throws synchronously. + var ex = Record.Exception(() => pump.AddCredit()); + + // The exception propagates to the caller (no catch in PerformRead). + Assert.NotNull(ex); + Assert.IsType(ex); + Assert.Equal("sync throw from ReadAsync", ex.Message); + } +} diff --git a/src/TurboHTTP.Tests/Protocol/Body/BodyPumpHelperSpec.cs b/src/TurboHTTP.Tests/Protocol/Body/BodyPumpHelperSpec.cs deleted file mode 100644 index 8d40c81ba..000000000 --- a/src/TurboHTTP.Tests/Protocol/Body/BodyPumpHelperSpec.cs +++ /dev/null @@ -1,144 +0,0 @@ -using Akka.Actor; -using TurboHTTP.Protocol.Body; - -namespace TurboHTTP.Tests.Protocol.Body; - -public sealed class BodyPumpHelperSpec -{ - private static BodyDrainSlot MakeSlot(Stream bodyStream, int chunkSize = 64) - { - var slot = new BodyDrainSlot(); - slot.Initialize(1, bodyStream, null, CancellationToken.None, null); - slot.EnsureBuffer(chunkSize); - return slot; - } - - private static MemoryStream MakeBody(int size) - { - var data = new byte[size]; - for (var i = 0; i < size; i++) - { - data[i] = (byte)(i % 256); - } - - return new MemoryStream(data); - } - - [Fact(Timeout = 5000)] - public void StartRead_should_return_CompletedSynchronously_for_sync_stream() - { - var body = MakeBody(100); - var slot = MakeSlot(body, 256); - - var result = BodyPumpHelper.StartRead(slot, 256, ActorRefs.Nobody); - - Assert.Equal(BodyPumpHelper.ReadOutcome.CompletedSynchronously, result.Outcome); - Assert.Equal(100, result.BytesRead); - Assert.False(slot.IsReadInFlight); - Assert.Equal(1, slot.ConsecutiveSyncReads); - - slot.DisposeResources(); - } - - [Fact(Timeout = 5000)] - public void StartRead_should_return_CompletedSynchronously_with_zero_for_EOF() - { - var body = new MemoryStream([]); - var slot = MakeSlot(body, 64); - - var result = BodyPumpHelper.StartRead(slot, 64, ActorRefs.Nobody); - - Assert.Equal(BodyPumpHelper.ReadOutcome.CompletedSynchronously, result.Outcome); - Assert.Equal(0, result.BytesRead); - - slot.DisposeResources(); - } - - [Fact(Timeout = 5000)] - public void StartRead_should_return_Dispatched_for_async_stream() - { - // Use a never-completing stream to force the async path - var neverStream = new NeverCompletingStream(); - var slot = new BodyDrainSlot(); - slot.Initialize(2, neverStream, null, CancellationToken.None, null); - slot.EnsureBuffer(64); - - var result = BodyPumpHelper.StartRead(slot, 64, ActorRefs.Nobody); - - Assert.Equal(BodyPumpHelper.ReadOutcome.Dispatched, result.Outcome); - Assert.Equal(0, result.BytesRead); - // IsReadInFlight stays true — BeginRead() was called but not cleared (async path) - Assert.True(slot.IsReadInFlight); - - slot.DisposeResources(); - } - - /// Stream whose ReadAsync never completes synchronously. - private sealed class NeverCompletingStream : Stream - { - private readonly TaskCompletionSource _tcs = new(); - - public override bool CanRead => true; - public override bool CanSeek => false; - public override bool CanWrite => false; - public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } - public override void Flush() { } - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotSupportedException(); - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) - => new(_tcs.Task); - } - - [Fact(Timeout = 5000)] - public void StartRead_should_increment_sync_counter_on_each_sync_read() - { - // StartRead increments ConsecutiveSyncReads on each sync completion. - // The caller (pump) checks the counter before calling StartRead and yields - // when it reaches MaxSyncReadsPerDispatch. - var body = MakeBody(BodyPumpHelper.MaxSyncReadsPerDispatch * 16 + 16); - var slot = MakeSlot(body, 16); - - for (var i = 0; i < BodyPumpHelper.MaxSyncReadsPerDispatch; i++) - { - var result = BodyPumpHelper.StartRead(slot, 16, ActorRefs.Nobody); - Assert.Equal(BodyPumpHelper.ReadOutcome.CompletedSynchronously, result.Outcome); - Assert.Equal(16, result.BytesRead); - Assert.Equal(i + 1, slot.ConsecutiveSyncReads); - } - - // After MaxSyncReadsPerDispatch reads, counter equals the threshold. - // Caller should now yield instead of calling StartRead again. - Assert.Equal(BodyPumpHelper.MaxSyncReadsPerDispatch, slot.ConsecutiveSyncReads); - - slot.DisposeResources(); - } - - [Fact(Timeout = 5000)] - public void StartRead_should_resume_cleanly_after_counter_reset() - { - var body = MakeBody((BodyPumpHelper.MaxSyncReadsPerDispatch + 1) * 16); - var slot = MakeSlot(body, 16); - - // Drive to threshold - for (var i = 0; i < BodyPumpHelper.MaxSyncReadsPerDispatch; i++) - { - BodyPumpHelper.StartRead(slot, 16, ActorRefs.Nobody); - } - - Assert.Equal(BodyPumpHelper.MaxSyncReadsPerDispatch, slot.ConsecutiveSyncReads); - - // Simulate what the pump does on yield: reset counter, then resume - slot.ResetSyncReads(); - Assert.Equal(0, slot.ConsecutiveSyncReads); - - var next = BodyPumpHelper.StartRead(slot, 16, ActorRefs.Nobody); - Assert.Equal(BodyPumpHelper.ReadOutcome.CompletedSynchronously, next.Outcome); - Assert.Equal(1, slot.ConsecutiveSyncReads); - - slot.DisposeResources(); - } -} diff --git a/src/TurboHTTP.Tests/Protocol/Body/FlowControlledBodyPumpSpec.cs b/src/TurboHTTP.Tests/Protocol/Body/FlowControlledBodyPumpSpec.cs index d19feaaf5..2c8848c39 100644 --- a/src/TurboHTTP.Tests/Protocol/Body/FlowControlledBodyPumpSpec.cs +++ b/src/TurboHTTP.Tests/Protocol/Body/FlowControlledBodyPumpSpec.cs @@ -1,4 +1,3 @@ -using System.Buffers; using Akka.Actor; using TurboHTTP.Pooling; using TurboHTTP.Protocol.Body; @@ -13,7 +12,9 @@ private sealed class FakeTarget : IBodyDrainTarget public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; public List Completed { get; } = []; public List<(int StreamId, Exception Reason)> Failed { get; } = []; - public IActorRef StageActor { get; } = ActorRefs.Nobody; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand => false; + public int PreferredChunkSize => 16 * 1024; public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -46,15 +47,18 @@ private static MemoryStream MakeBody(int size) return new MemoryStream(data); } + private static FlowControlledBodyPump MakePump(FakeTarget target, FlowController flow) + => new(target, flow, new ConnectionPoolContext(), new CancellationTokenSource()); + [Fact(Timeout = 5000)] public void Register_should_emit_body_when_window_available() { var target = new FakeTarget(); var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); - pump.Register(1, MakeBody(100), 100, CancellationToken.None); + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); Assert.Equal(2, target.Emitted.Count); Assert.Equal(100, target.Emitted[0].Data.Length); @@ -68,17 +72,18 @@ public void Register_should_block_when_stream_window_zero() { var target = new FakeTarget(); var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); flow.OnDataSent(1, 65535); - // Stream window is now 0 - pump.Register(1, MakeBody(100), 100, CancellationToken.None); + // Stream window is now 0, connection window reduced too + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); Assert.Empty(target.Emitted); Assert.Empty(target.Completed); - // WINDOW_UPDATE for stream 1 + // WINDOW_UPDATE for both connection and stream 1 + flow.OnSendWindowUpdate(0, 65535); flow.OnSendWindowUpdate(1, 65535); pump.OnWindowUpdate(1); @@ -87,31 +92,30 @@ public void Register_should_block_when_stream_window_zero() } [Fact(Timeout = 5000)] - public void PartialSend_should_store_limbo_and_drain_on_window_update() + public void Register_should_block_when_window_below_half_chunk_size() { var target = new FakeTarget(); - var flow = MakeFlow(connWindow: 65535); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); - // Exhaust most of the window - flow.OnDataSent(1, 65535 - 50); - // Stream window = 50, conn window = 50 + // Exhaust stream window to below chunkSize/2 (= 8192) + // Leave only 4096 bytes (less than 8192 threshold) + flow.OnDataSent(1, 65535 - 4 * 1024); + // Stream window = 4096, conn window still large - pump.Register(1, MakeBody(200), 200, CancellationToken.None); + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); - // Should send 50 bytes, limbo 150 - Assert.Single(target.Emitted); - Assert.Equal(50, target.Emitted[0].Data.Length); + // Stream is window-blocked: 4096 < 8192 + Assert.Empty(target.Emitted); + Assert.Empty(target.Completed); - // WINDOW_UPDATE - flow.OnSendWindowUpdate(1, 200); - flow.OnSendWindowUpdate(0, 200); + // Open window above threshold + flow.OnSendWindowUpdate(1, 32 * 1024); + flow.OnSendWindowUpdate(0, 32 * 1024); pump.OnWindowUpdate(1); - // Should drain limbo (150) + read more + EOF - var totalData = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); - Assert.Equal(200, totalData); + Assert.Equal(2, target.Emitted.Count); Assert.Single(target.Completed); } @@ -120,19 +124,19 @@ public void RoundRobin_should_interleave_two_streams() { var target = new FakeTarget(); var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 64, hardCap: 16); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); flow.InitStreamSendWindow(3); - pump.Register(1, MakeBody(128), 128, CancellationToken.None); - pump.Register(3, MakeBody(128), 128, CancellationToken.None); + pump.Register(1, MakeBody(128), CancellationToken.None, initialCredits: 16); + pump.Register(3, MakeBody(128), CancellationToken.None, initialCredits: 16); // Both should complete (sync fast path) Assert.Contains(1, target.Completed); Assert.Contains(3, target.Completed); - // Verify interleaving: stream IDs should alternate + // Verify both streams emitted data var streamIds = target.Emitted.Where(e => !e.EndStream).Select(e => e.StreamId).ToList(); Assert.Contains(1, streamIds); Assert.Contains(3, streamIds); @@ -143,10 +147,10 @@ public void Orphan_should_not_crash_on_callback_after_cancel() { var target = new FakeTarget(); var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); - pump.Register(1, MakeBody(100), 100, CancellationToken.None); + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); // Already completed for sync stream pump.Cancel(1); @@ -158,7 +162,7 @@ public void Cleanup_should_be_idempotent() { var target = new FakeTarget(); var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var pump = MakePump(target, flow); pump.Cleanup(); pump.Cleanup(); @@ -169,10 +173,10 @@ public void SyncFastPath_should_drain_without_PipeTo() { var target = new FakeTarget(); var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); - pump.Register(1, MakeBody(100), 100, CancellationToken.None); + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); // MemoryStream completes synchronously — no PipeTo needed Assert.Equal(2, target.Emitted.Count); @@ -180,100 +184,660 @@ public void SyncFastPath_should_drain_without_PipeTo() } [Fact(Timeout = 5000)] - public void SyncStarvationGuard_should_yield_after_64_reads() + public void Sync_reads_should_complete_all_chunks() { + // Pump should drain all chunks without starvation. var target = new FakeTarget(); var flow = MakeFlow(connWindow: 1024 * 1024); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 16, hardCap: 16); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); flow.OnSendWindowUpdate(1, 1024 * 1024); - pump.Register(1, MakeBody(65 * 16), 65 * 16, CancellationToken.None); + var bodySize = 65 * 16; + pump.Register(1, MakeBody(bodySize), CancellationToken.None, initialCredits: 16); + + // All bytes emitted + EOF (no starvation guard) + var totalBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); + Assert.Equal(bodySize, totalBytes); + Assert.Single(target.Completed); + } + + [Fact(Timeout = 5000)] + public void SlotPooling_should_reuse_slots() + { + var target = new FakeTarget(); + var flow = MakeFlow(); + var pump = MakePump(target, flow); + + flow.InitStreamSendWindow(1); + pump.Register(1, MakeBody(10), CancellationToken.None, initialCredits: 16); + Assert.Single(target.Completed); + + // Register again — should reuse pooled slot + flow.InitStreamSendWindow(3); + target.Completed.Clear(); + pump.Register(3, MakeBody(10), CancellationToken.None, initialCredits: 16); + Assert.Single(target.Completed); + } + + [Fact(Timeout = 5000)] + public void CtsLifecycle_should_create_once_and_dispose_on_complete() + { + var target = new FakeTarget(); + var flow = MakeFlow(); + var connCts = new CancellationTokenSource(); + var reqCts = new CancellationTokenSource(); + var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), connCts); + + flow.InitStreamSendWindow(1); + pump.Register(1, MakeBody(100), reqCts.Token, initialCredits: 16); - var emittedBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); - Assert.Equal(64 * 16, emittedBytes); + Assert.Single(target.Completed); + reqCts.Dispose(); + } + + [Fact(Timeout = 5000)] + public void ReadRound_should_reserve_window_before_read() + { + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + flow.InitStreamSendWindow(1); + var initialConnWindow = flow.ConnectionSendWindow; + var initialStreamWindow = flow.GetStreamSendWindow(1); + + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); + + // After completing the drain, windows should have been decremented and refunded. + // Net effect: 100 bytes were effectively consumed (not refunded). + // connWindow reduced by 100 bytes net (reserved 16384, refunded 16284 after first read of 100 bytes, + // then reserved again for the EOF read and fully refunded). + var netConnChange = initialConnWindow - flow.ConnectionSendWindow; + Assert.True(netConnChange >= 0, "Window should not increase beyond initial."); + Assert.Single(target.Completed); + } + + [Fact(Timeout = 5000)] + public void ReadRound_should_skip_stream_when_window_below_half_chunksize() + { + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + flow.InitStreamSendWindow(1); + // Set stream window to 1 byte (below chunkSize/2 = 8192) + flow.OnDataSent(1, 65534); + // Stream window = 1, far below the 8192 threshold + + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); + + // No reads should happen — stream is window-blocked + Assert.Empty(target.Emitted); Assert.Empty(target.Completed); + } - // Guard fires DrainContinue(1) to StageActor (Nobody in tests — message dropped). - // Verified behaviorally: pump stops at exactly 64 chunks, proving the guard path was taken. - pump.HandleDrainContinue(1); + [Fact(Timeout = 5000)] + public void OnWindowUpdate_should_unblock_stream_and_trigger_read_with_credits() + { + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); - var totalBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); - Assert.Equal(65 * 16, totalBytes); + flow.InitStreamSendWindow(1); + // Block the stream below threshold + flow.OnDataSent(1, 65534); + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + Assert.Empty(target.Emitted); + + // Restore window above threshold and signal update + flow.OnSendWindowUpdate(1, 65534); + flow.OnSendWindowUpdate(0, 65534); + pump.OnWindowUpdate(1); + + // Stream should now be unblocked and drained + Assert.NotEmpty(target.Emitted); Assert.Single(target.Completed); } [Fact(Timeout = 5000)] - public void ConnectionWindowCeiling_should_cap_effectiveSlots() + public void AfterRead_should_refund_unused_reservation() { + // Arrange: set up a stream that reads less than the full reserved window var target = new FakeTarget(); - // Start with very small connection window - var flow = MakeFlow(connWindow: 65535); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 16 * 1024, hardCap: 16); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); - // Connection window = 65535, chunkSize = 16384 - // effectiveSlots = min(readSlots=2, 65535/16384=4, 16) = 2 - // This is a self-consistency test — the pump should not over-read flow.InitStreamSendWindow(1); - pump.Register(1, MakeBody(100), 100, CancellationToken.None); + var windowBefore = flow.ConnectionSendWindow; + + // Register a body smaller than chunkSize so the reservation exceeds bytes read + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); + + var windowAfter = flow.ConnectionSendWindow; + // Net window consumed should be exactly 100 bytes (reserved 16384 per read, + // refunded 16284 after reading 100 bytes, then reserved 16384 for EOF read and fully refunded) + var netConsumed = windowBefore - windowAfter; + // After full drain including EOF read, net consumption is 0 (all data bytes already + // charged via OnDataSent pattern — but here Reserve/Refund are used, not OnDataSent, + // so the pump manages the deduction). + // Exact value depends on read sequence; we verify consistency only: + Assert.True(netConsumed >= 0, "Refund should not leave window higher than before registration."); Assert.Single(target.Completed); } + // H2 window reservation integration + [Fact(Timeout = 5000)] - public void RegisterWithLimbo_should_drain_on_connection_window_update() + public void Register_should_decrement_flow_controller_window_during_read() { + // Full cycle: register body → credits → pump reads with reservation → + // verify FlowController windows decremented → drain complete. var target = new FakeTarget(); - var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + flow.InitStreamSendWindow(1); + var connWindowBefore = flow.ConnectionSendWindow; + var streamWindowBefore = flow.GetStreamSendWindow(1); + + // Register and drain a 100-byte body. + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); + + // Windows should have been decremented by the reservation, then refunded for the unused portion. + // Net: they should be <= original (refund restores unused reservation, but actual data was reserved). + Assert.True(flow.ConnectionSendWindow <= connWindowBefore, + "Connection send window should not exceed initial after reservation."); + Assert.True(flow.GetStreamSendWindow(1) <= streamWindowBefore, + "Stream send window should not exceed initial after reservation."); + Assert.Single(target.Completed); + } + + [Fact(Timeout = 5000)] + public void Register_should_refund_unused_reservation_exactly() + { + // After draining a 100-byte body (< chunkSize = 16384): + // BeforeRead reserves 16384, AfterRead refunds (16384 - 100) = 16284. + // Net deduction per data read = 100. + // EOF read: reserves 16384, reads 0, refunds 16384. Net = 0. + // Total net = 100 bytes consumed from both windows. + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); flow.InitStreamSendWindow(1); - // Exhaust connection window + var connWindowBefore = flow.ConnectionSendWindow; + var streamWindowBefore = flow.GetStreamSendWindow(1); + + pump.Register(1, MakeBody(100), CancellationToken.None, initialCredits: 16); + + var connWindowAfter = flow.ConnectionSendWindow; + var streamWindowAfter = flow.GetStreamSendWindow(1); + + // Net deduction from each window should be exactly 100 bytes (the data read). + Assert.Equal(100, connWindowBefore - connWindowAfter); + Assert.Equal(100, streamWindowBefore - streamWindowAfter); + Assert.Single(target.Completed); + } + + [Fact(Timeout = 5000)] + public void WindowUpdate_full_cycle_should_unblock_and_complete_drain() + { + // Full integration cycle: + // 1. Register body + // 2. Block by exhausting stream window + // 3. Verify no reads + // 4. WINDOW_UPDATE arrives (both conn and stream) + // 5. Verify reads complete + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + // Exhaust the stream window + flow.InitStreamSendWindow(1); flow.OnDataSent(1, 65535); - var remainder = new byte[] { 1, 2, 3, 4, 5 }; - pump.RegisterWithLimbo(1, remainder, CancellationToken.None); + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + // No reads — stream blocked Assert.Empty(target.Emitted); + // Restore windows flow.OnSendWindowUpdate(0, 65535); + flow.OnSendWindowUpdate(1, 65535); + pump.OnWindowUpdate(1); + + // Should now drain + Assert.Equal(2, target.Emitted.Count); + Assert.Single(target.Completed); + } + + // WINDOW_UPDATE deadlock prevention + + [Fact(Timeout = 5000)] + public void OnWindowUpdate_connection_level_should_unblock_all_eligible_streams() + { + // Connection-level WINDOW_UPDATE (streamId == 0) should re-evaluate ALL blocked streams + // and unblock those with sufficient per-stream window. + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + // Two streams both blocked (stream window exhausted by OnDataSent). + flow.InitStreamSendWindow(1); + flow.InitStreamSendWindow(3); + flow.OnDataSent(1, 65535); + flow.OnDataSent(3, 65535); + + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + pump.Register(3, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // Neither should have emitted (blocked before bootstrap credits could run, or blocked after). + Assert.Empty(target.Completed); + + // Restore per-stream windows above threshold (chunkSize/2 = 8192). + flow.OnSendWindowUpdate(1, 65535); + flow.OnSendWindowUpdate(3, 65535); + + // Also restore connection window and issue connection-level update (streamId == 0). + // This triggers the bulk re-evaluation path in OnWindowUpdate. + flow.OnSendWindowUpdate(0, 65535 * 2); pump.OnWindowUpdate(0); - Assert.Single(target.Emitted); - Assert.Equal(5, target.Emitted[0].Data.Length); + // Both streams should drain. + Assert.Contains(1, target.Completed); + Assert.Contains(3, target.Completed); } [Fact(Timeout = 5000)] - public void SlotPooling_should_reuse_slots() + public void OnWindowUpdate_all_blocked_with_credits_should_trigger_reads() { + // All streams blocked (window-blocked), credits accumulated, WINDOW_UPDATE arrives → reads trigger. var target = new FakeTarget(); - var flow = MakeFlow(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024, hardCap: 16); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + // Block two streams flow.InitStreamSendWindow(1); - pump.Register(1, MakeBody(10), 10, CancellationToken.None); - Assert.Single(target.Completed); + flow.InitStreamSendWindow(3); + flow.OnDataSent(1, 65535); + flow.OnDataSent(3, 65535); - // Register again — should reuse pooled slot + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + pump.Register(3, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // Both blocked: no emits beyond bootstrap (bootstrap credits may have been spent trying to read) + Assert.Empty(target.Completed); + + // Now give both streams and conn window enough room + flow.OnSendWindowUpdate(0, 65535 * 2); + flow.OnSendWindowUpdate(1, 65535); + flow.OnSendWindowUpdate(3, 65535); + + // OnWindowUpdate for stream 1 — should trigger reads for stream 1 (and potentially 3). + pump.OnWindowUpdate(1); + pump.OnWindowUpdate(3); + + // Both small streams should drain. + Assert.Contains(1, target.Completed); + Assert.Contains(3, target.Completed); + } + + // Cancellation — cancel of window-blocked stream + + [Fact(Timeout = 5000)] + public void Cancel_window_blocked_stream_should_remove_from_blocked_set() + { + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + // Block stream 1 + flow.InitStreamSendWindow(1); + flow.OnDataSent(1, 65535); + + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // Stream is window-blocked. + Assert.Empty(target.Emitted); + + // Cancel the stream. + pump.Cancel(1); + + // After cancel, a WINDOW_UPDATE for stream 1 should not unblock it or emit anything. + flow.OnSendWindowUpdate(0, 65535); + flow.OnSendWindowUpdate(1, 65535); + pump.OnWindowUpdate(1); + + // No data should have been emitted for stream 1. + Assert.DoesNotContain(target.Emitted, e => e.StreamId == 1 && !e.EndStream); + Assert.Empty(target.Failed); + } + + [Fact(Timeout = 5000)] + public void CancelAll_with_window_blocked_streams_should_clear_blocked_set() + { + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var cts = new CancellationTokenSource(); + var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), cts); + + flow.InitStreamSendWindow(1); flow.InitStreamSendWindow(3); - target.Completed.Clear(); - pump.Register(3, MakeBody(10), 10, CancellationToken.None); + flow.OnDataSent(1, 65535); + flow.OnDataSent(3, 65535); + + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + pump.Register(3, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // CancelAll should clean up including window-blocked streams. + pump.CancelAll(); + + Assert.True(cts.IsCancellationRequested); + + // After CancelAll, restoring windows and sending updates should not trigger any reads. + flow.OnSendWindowUpdate(0, 65535 * 2); + flow.OnSendWindowUpdate(1, 65535); + flow.OnSendWindowUpdate(3, 65535); + pump.OnWindowUpdate(1); + pump.OnWindowUpdate(3); + + Assert.Empty(target.Emitted); + } + + // Regression: Bug 3 — OnWindowUpdate with zero credits (FlowControlledBodyPump) + // Before the fix: OnWindowUpdate guard was `GetCredits() > 0`. When all streams were + // window-blocked and credits were at 0, WINDOW_UPDATE would unblock streams (via + // _windowBlockedStreams.Remove + EnqueueStream) but skip the credit boost, leaving + // streams in the ready queue with 0 credits → permanent stall. + // Fix: guard changed to `GetActiveStreamCount() > 0` — injects credits whenever any + // active stream exists, regardless of current credit level. + + // A dedicated async stream to drain credits before the window-update scenario. + // ReadAsync returns a non-completed ValueTask on the first call. Because FakeTarget's + // PipeToTarget is ActorRefs.Nobody, the PipeTo message is dropped and the pump must be + // driven forward by an explicit HandleReadComplete call. + private sealed class OnceAsyncStream : Stream + { + private readonly byte[] _data; + private int _position; + private bool _firstRead = true; + + public OnceAsyncStream(byte[] data) => _data = data; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (_firstRead) + { + _firstRead = false; + // Advance internal position immediately so subsequent sync reads see the correct + // stream position — the bytes are considered read even though the ValueTask is + // returned as non-completed (to force the async dispatch path in BodyPumpBase + // so that no sync credit reclaim occurs). + var n = ReadSync(buffer); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken)); + // TCS never resolves — PipeTo goes to Nobody, test drives via HandleReadComplete. + return new ValueTask(tcs.Task); + } + + return ValueTask.FromResult(ReadSync(buffer)); + } + + private int ReadSync(Memory buffer) + { + var count = Math.Min(buffer.Length, _data.Length - _position); + if (count == 0) + { + return 0; + } + + _data.AsSpan(_position, count).CopyTo(buffer.Span); + _position += count; + return count; + } + } + + [Fact(Timeout = 5000)] + public void OnWindowUpdate_should_inject_credits_and_unblock_stream_even_when_credits_were_zero() + { + // Arrange: a stream that first goes async (draining 1 credit with no reclaim) so that + // credit level reaches 0, then becomes window-blocked. We then verify that a + // WINDOW_UPDATE triggers reads via the GetActiveStreamCount() guard. + // + // Bug scenario: old guard GetCredits() > 0 would fail when credits = 0 → + // the stream stays in the ready queue but nothing drives reads → permanent stall. + // Fix: GetActiveStreamCount() > 0 injects credits unconditionally when any stream exists. + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + // Stream 1: window large enough for one read. The body uses OnceAsyncStream so the first + // read goes async, consuming a credit without reclaiming it. + flow.InitStreamSendWindow(1); + flow.OnSendWindowUpdate(1, 1024 * 1024); + + var data = new byte[100]; + var body = new OnceAsyncStream(data); + + pump.Register(1, body, CancellationToken.None, initialCredits: 16); + + // At this point the first read was dispatched asynchronously (slot is in-flight). + // Simulate delivery of the async read result: 100 bytes read. + // HandleReadComplete advances the stream to an EOF read (sync) and completes the drain. + pump.HandleReadComplete(1, 100); + + // The drain must complete: data + endStream emitted, OnDrainComplete called. Assert.Single(target.Completed); + Assert.Contains(target.Emitted, e => e.EndStream); } [Fact(Timeout = 5000)] - public void CtsLifecycle_should_create_once_and_dispose_on_complete() + public void OnWindowUpdate_should_resume_window_blocked_stream_regardless_of_credit_level() { + // Arrange: a stream that is immediately window-blocked (stream window < chunkSize/2). + // Bootstrap credits are injected by Register but the stream goes into windowBlockedStreams. + // We then restore the window and call OnWindowUpdate — verifies that the pump unblocks + // the stream and injects fresh credits even if the credit guard were checking 0. + // + // This is the direct observable consequence of Bug 3's fix: the GetActiveStreamCount() + // guard ensures credits are always injected when streams exist, not just when credits > 0. var target = new FakeTarget(); - var flow = MakeFlow(); - var connCts = new CancellationTokenSource(); - var reqCts = new CancellationTokenSource(); - var pump = new FlowControlledBodyPump(target, flow, new ConnectionPoolContext(), connCts, chunkSize: 1024, hardCap: 16); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + // Init stream and immediately exhaust it — stream window = 0 < threshold (8192). flow.InitStreamSendWindow(1); - pump.Register(1, MakeBody(100), 100, reqCts.Token); + flow.OnDataSent(1, 65535); + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // Stream is window-blocked: bootstrap credits accumulated but no reads fired. + Assert.Empty(target.Emitted); + + // Restore windows and send WINDOW_UPDATE. The fix ensures credit injection + // always runs when active streams exist. + flow.OnSendWindowUpdate(0, 65535 * 2); + flow.OnSendWindowUpdate(1, 65535); + pump.OnWindowUpdate(1); + + // Stream should unblock and drain. Assert.Single(target.Completed); - reqCts.Dispose(); + } + + // Regression: Bug 4 — window reservation leak on orphaned/failed reads + // Before the fix: when a read was orphaned (stream cancelled while read was in-flight), + // HandleReadComplete called CleanupSlot without calling AfterRead first. The reserved + // window (from BeforeRead) was never refunded → connection send window leaked permanently. + // Fix: call AfterRead(streamId, slot, 0) before CleanupSlot on the orphaned path. + + [Fact(Timeout = 5000)] + public void HandleReadComplete_should_refund_window_reservation_when_stream_was_cancelled_during_read() + { + // Arrange: a stream using OnceAsyncStream so the first read goes async (in-flight). + // We cancel the stream while the read is in-flight, then deliver the async completion. + // Fix: AfterRead is called before CleanupSlot on the orphaned path, refunding the + // reserved window. Without the fix, the reservation leaks and the connection window + // is permanently reduced by chunkSize (16384 bytes). + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + flow.InitStreamSendWindow(1); + flow.OnSendWindowUpdate(1, 1024 * 1024); + + var connWindowBefore = flow.ConnectionSendWindow; + + var data = new byte[16 * 1024]; + var body = new OnceAsyncStream(data); + + pump.Register(1, body, CancellationToken.None, initialCredits: 16); + + // First read is async (in-flight). BeforeRead reserved chunkSize (16384) from the window. + // Cancel the stream while the read is still in-flight: slot becomes orphaned. + pump.Cancel(1); + + // Now deliver the async read completion. The orphaned path must call AfterRead to + // refund the reserved window before cleaning up the slot. + pump.HandleReadComplete(1, 16 * 1024); + + // The connection window must be fully restored to its pre-read level. + // Bug: without AfterRead on orphan, the 16384-byte reservation is never refunded. + // Fix: AfterRead(streamId, slot, 0) refunds the full reservation before cleanup. + Assert.Equal(connWindowBefore, flow.ConnectionSendWindow); + } + + // Edge case: multi-stream connection window contention + + [Fact(Timeout = 5000)] + public void Register_fourth_stream_should_be_window_blocked_when_connection_window_exhausted_by_three_reads() + { + // Connection window holds exactly 48 KB. Three streams each consume 16 KB (one chunkSize + // reservation each). When the 4th stream tries to read, the connection window is below + // chunkSize/2 = 8 KB and the stream goes into _windowBlockedStreams. + var target = new FakeTarget(); + + // Set connection window to exactly 48 KB = 3 * 16 KB. + var flow = new FlowController(1024 * 1024, 64 * 1024); + flow.OnSendWindowUpdate(0, 3 * 16 * 1024 - 65535); // adjust from initial 65535 to 48 KB + var pump = MakePump(target, flow); + + // Give each stream a large per-stream window so only the connection window is the bottleneck. + for (var id = 1; id <= 4; id++) + { + flow.InitStreamSendWindow(id); + flow.OnSendWindowUpdate(id, 1024 * 1024); + } + + // Register 4 streams — each Register bootstraps 16 credits. + // The first 3 streams should each read one 16-KB chunk, consuming the full connection window. + // The 4th stream should be window-blocked because connection window < chunkSize/2. + pump.Register(1, MakeBody(32 * 1024), CancellationToken.None, initialCredits: 16); + pump.Register(2, MakeBody(32 * 1024), CancellationToken.None, initialCredits: 16); + pump.Register(3, MakeBody(32 * 1024), CancellationToken.None, initialCredits: 16); + pump.Register(4, MakeBody(32 * 1024), CancellationToken.None, initialCredits: 16); + + // Stream 4 must not have any data emitted (window-blocked). + var stream4DataEmits = target.Emitted.Count(e => e.StreamId == 4 && !e.EndStream); + Assert.Equal(0, stream4DataEmits); + + // Restore connection window and send connection-level WINDOW_UPDATE to unblock. + flow.OnSendWindowUpdate(0, 4 * 1024 * 1024); + pump.OnWindowUpdate(0); + + // After unblocking, stream 4 should eventually drain. + Assert.Contains(4, target.Completed); + } + + // Edge case: OnWindowUpdate for cancelled stream + + [Fact(Timeout = 5000)] + public void OnWindowUpdate_for_cancelled_stream_should_not_throw_or_reenqueue() + { + // Block a stream, cancel it (removes from _windowBlockedStreams via OnStreamCancelled), + // then call OnWindowUpdate for that streamId. Must be a safe no-op. + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + // Block stream 5 by exhausting its send window. + flow.InitStreamSendWindow(5); + flow.OnDataSent(5, 65535); + + pump.Register(5, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // Stream 5 is window-blocked. + Assert.Empty(target.Emitted); + + // Cancel the stream — this removes it from _windowBlockedStreams. + pump.Cancel(5); + + // Restore window and signal update for the now-cancelled stream 5. + flow.OnSendWindowUpdate(0, 65535); + flow.OnSendWindowUpdate(5, 65535); + + var ex = Record.Exception(() => pump.OnWindowUpdate(5)); + Assert.Null(ex); + + // The cancelled stream must not have been re-enqueued or caused any emission. + Assert.Empty(target.Emitted); + Assert.Empty(target.Failed); + } + + // Edge case: connection-level WINDOW_UPDATE with mixed blocked/cancelled streams + + [Fact(Timeout = 5000)] + public void OnWindowUpdate_connection_level_should_unblock_only_non_cancelled_blocked_streams() + { + // Three streams are window-blocked. One is then cancelled. + // A connection-level WINDOW_UPDATE (streamId == 0) should unblock only the 2 + // non-cancelled streams — the cancelled one was removed from _windowBlockedStreams + // by OnStreamCancelled and must not receive any emissions. + var target = new FakeTarget(); + var flow = MakeFlow(connWindow: 1024 * 1024); + var pump = MakePump(target, flow); + + // Block all three streams by exhausting per-stream send windows. + for (var id = 1; id <= 3; id++) + { + flow.InitStreamSendWindow(id); + flow.OnDataSent(id, 65535); + } + + pump.Register(1, MakeBody(50), CancellationToken.None, initialCredits: 16); + pump.Register(2, MakeBody(50), CancellationToken.None, initialCredits: 16); + pump.Register(3, MakeBody(50), CancellationToken.None, initialCredits: 16); + + // All three blocked — nothing emitted yet. + Assert.Empty(target.Completed); + + // Cancel stream 2 while it's window-blocked. + pump.Cancel(2); + + // Restore per-stream windows so all would be eligible after unblocking. + flow.OnSendWindowUpdate(1, 65535); + flow.OnSendWindowUpdate(2, 65535); + flow.OnSendWindowUpdate(3, 65535); + + // Issue a connection-level WINDOW_UPDATE (streamId == 0). + flow.OnSendWindowUpdate(0, 65535 * 3); + pump.OnWindowUpdate(0); + + // Streams 1 and 3 must drain; stream 2 must not emit anything. + Assert.Contains(1, target.Completed); + Assert.Contains(3, target.Completed); + Assert.DoesNotContain(target.Emitted, e => e.StreamId == 2 && !e.EndStream); } } diff --git a/src/TurboHTTP.Tests/Protocol/Body/MultiplexedBodyPumpSpec.cs b/src/TurboHTTP.Tests/Protocol/Body/MultiplexedBodyPumpSpec.cs index 9e3a307e7..118c66b82 100644 --- a/src/TurboHTTP.Tests/Protocol/Body/MultiplexedBodyPumpSpec.cs +++ b/src/TurboHTTP.Tests/Protocol/Body/MultiplexedBodyPumpSpec.cs @@ -1,4 +1,3 @@ -using System.Buffers; using Akka.Actor; using TurboHTTP.Pooling; using TurboHTTP.Protocol.Body; @@ -12,7 +11,9 @@ private sealed class FakeTarget : IBodyDrainTarget public List<(long StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; public List Completed { get; } = []; public List<(long StreamId, Exception Reason)> Failed { get; } = []; - public IActorRef StageActor { get; } = ActorRefs.Nobody; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand => false; + public int PreferredChunkSize => 16 * 1024; public void EmitDataFrames(long streamId, ReadOnlyMemory data, bool endStream) { @@ -38,9 +39,9 @@ private static MemoryStream MakeBody(int size) public void Register_should_emit_body_immediately() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, MakeBody(100), 100, CancellationToken.None); + pump.Register(1L, MakeBody(100), CancellationToken.None, initialCredits: 16); Assert.Equal(2, target.Emitted.Count); Assert.Equal(100, target.Emitted[0].Data.Length); @@ -52,10 +53,10 @@ public void Register_should_emit_body_immediately() public void RoundRobin_should_interleave_streams() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 64); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, MakeBody(128), 128, CancellationToken.None); - pump.Register(3L, MakeBody(128), 128, CancellationToken.None); + pump.Register(1L, MakeBody(128), CancellationToken.None, initialCredits: 16); + pump.Register(3L, MakeBody(128), CancellationToken.None, initialCredits: 16); Assert.Contains(1L, target.Completed); Assert.Contains(3L, target.Completed); @@ -65,9 +66,9 @@ public void RoundRobin_should_interleave_streams() public void Cancel_should_handle_orphan() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, MakeBody(100), 100, CancellationToken.None); + pump.Register(1L, MakeBody(100), CancellationToken.None, initialCredits: 16); pump.Cancel(1L); } @@ -75,7 +76,7 @@ public void Cancel_should_handle_orphan() public void Cleanup_should_be_idempotent() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); pump.Cleanup(); pump.Cleanup(); @@ -85,27 +86,23 @@ public void Cleanup_should_be_idempotent() public void SyncFastPath_should_drain_inline() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, MakeBody(50), 50, CancellationToken.None); + pump.Register(1L, MakeBody(50), CancellationToken.None, initialCredits: 16); Assert.Equal(2, target.Emitted.Count); Assert.Single(target.Completed); } [Fact(Timeout = 5000)] - public void SyncStarvationGuard_should_yield() + public void Sync_reads_should_complete_without_starvation_guard() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 16); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, MakeBody(65 * 16), 65 * 16, CancellationToken.None); - - var emitted = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); - Assert.Equal(64 * 16, emitted); - - pump.HandleDrainContinue(1L); + pump.Register(1L, MakeBody(65 * 16), CancellationToken.None, initialCredits: 16); + // All chunks emitted + EOF var total = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); Assert.Equal(65 * 16, total); Assert.Single(target.Completed); @@ -115,13 +112,13 @@ public void SyncStarvationGuard_should_yield() public void SlotPooling_should_reuse_after_drain() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, MakeBody(10), 10, CancellationToken.None); + pump.Register(1L, MakeBody(10), CancellationToken.None, initialCredits: 16); Assert.Single(target.Completed); target.Completed.Clear(); - pump.Register(3L, MakeBody(10), 10, CancellationToken.None); + pump.Register(3L, MakeBody(10), CancellationToken.None, initialCredits: 16); Assert.Single(target.Completed); } @@ -129,9 +126,9 @@ public void SlotPooling_should_reuse_after_drain() public void EOF_should_emit_endStream() { var target = new FakeTarget(); - var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource(), chunkSize: 1024); + var pump = new MultiplexedBodyPump(target, new ConnectionPoolContext(), new CancellationTokenSource()); - pump.Register(1L, new MemoryStream([]), 0, CancellationToken.None); + pump.Register(1L, new MemoryStream([]), CancellationToken.None, initialCredits: 16); Assert.Single(target.Emitted); Assert.True(target.Emitted[0].EndStream); diff --git a/src/TurboHTTP.Tests/Protocol/Body/SerialBodyPumpSpec.cs b/src/TurboHTTP.Tests/Protocol/Body/SerialBodyPumpSpec.cs index 6b8fb0a36..1f8a9dc50 100644 --- a/src/TurboHTTP.Tests/Protocol/Body/SerialBodyPumpSpec.cs +++ b/src/TurboHTTP.Tests/Protocol/Body/SerialBodyPumpSpec.cs @@ -1,5 +1,7 @@ using System.Buffers; +using System.IO.Pipelines; using Akka.Actor; +using TurboHTTP.Pooling; using TurboHTTP.Protocol.Body; namespace TurboHTTP.Tests.Protocol.Body; @@ -11,7 +13,9 @@ private sealed class FakeTarget : IBodyDrainTarget public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; public List Completed { get; } = []; public List<(int StreamId, Exception Reason)> Failed { get; } = []; - public IActorRef StageActor { get; } = ActorRefs.Nobody; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand => false; + public int PreferredChunkSize => 16 * 1024; public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -23,7 +27,7 @@ public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStre } /// - /// Target that calls OnCapacityAvailable() synchronously after EmitDataFrames, + /// Target that calls AddCredit() synchronously after EmitDataFrames, /// simulating H1.0 behavior where the target drives the pump inline. /// private sealed class AutoResumeTarget : IBodyDrainTarget @@ -32,7 +36,9 @@ private sealed class AutoResumeTarget : IBodyDrainTarget public List<(int StreamId, byte[] Data, bool EndStream)> Emitted { get; } = []; public List Completed { get; } = []; public List<(int StreamId, Exception Reason)> Failed { get; } = []; - public IActorRef StageActor { get; } = ActorRefs.Nobody; + public IActorRef PipeToTarget { get; } = ActorRefs.Nobody; + public bool HasPendingDemand => false; + public int PreferredChunkSize => 16 * 1024; public void SetPump(SerialBodyPump pump) => _pump = pump; @@ -41,7 +47,7 @@ public void EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStre Emitted.Add((streamId, data.ToArray(), endStream)); if (!endStream) { - _pump?.OnCapacityAvailable(); + _pump?.AddCredit(); } } @@ -63,13 +69,13 @@ private static MemoryStream MakeBody(int size) [Fact(Timeout = 5000)] public void Register_should_emit_body_immediately_for_sync_stream() { - // maxCapacity=2: allows 2 reads in flight. A 100-byte body with 1024-byte chunk - // produces 1 data read (100 bytes) + 1 EOF read (0 bytes) = exactly 2 reads. var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 1024, maxCapacity: 2); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); var body = MakeBody(100); - pump.Register(body, 100, CancellationToken.None); + pump.Register(body, CancellationToken.None); Assert.Equal(2, target.Emitted.Count); Assert.Equal(100, target.Emitted[0].Data.Length); @@ -83,10 +89,12 @@ public void Register_should_emit_body_immediately_for_sync_stream() public void Register_should_emit_endStream_on_empty_body() { var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 1024, maxCapacity: 2); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); var body = new MemoryStream([]); - pump.Register(body, 0, CancellationToken.None); + pump.Register(body, CancellationToken.None); Assert.Single(target.Emitted); Assert.True(target.Emitted[0].EndStream); @@ -94,44 +102,43 @@ public void Register_should_emit_endStream_on_empty_body() } [Fact(Timeout = 5000)] - public void Register_should_chunk_large_body_with_auto_resume() + public void Register_should_emit_complete_body_with_auto_resume() { - // Large body with small chunks and maxCapacity=1 — needs AutoResumeTarget - // to call OnCapacityAvailable inline (H1.0 convention). + // Register with initial credits drains small body immediately. + // AutoResumeTarget is not actually needed now that we have initial credits. var target = new AutoResumeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 64, maxCapacity: 1); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); target.SetPump(pump); var body = MakeBody(200); - pump.Register(body, 200, CancellationToken.None); + pump.Register(body, CancellationToken.None); var dataEmits = target.Emitted.Where(e => !e.EndStream).ToList(); - Assert.True(dataEmits.Count >= 3); + Assert.Single(dataEmits); // 200 bytes < 16 KB chunk = 1 emit Assert.Equal(200, dataEmits.Sum(e => e.Data.Length)); Assert.True(target.Emitted[^1].EndStream); + Assert.Single(target.Completed); } [Fact(Timeout = 5000)] - public void OnCapacityAvailable_should_resume_after_capacity_exhaustion() + public void AddCredit_should_resume_after_budget_exhaustion() { var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 64, maxCapacity: 1); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); var body = MakeBody(200); - pump.Register(body, 200, CancellationToken.None); + pump.Register(body, CancellationToken.None); - // maxCapacity=1: pump emits 1 chunk then stops (FakeTarget doesn't auto-resume) - Assert.Single(target.Emitted, e => !e.EndStream); - Assert.Equal(64, target.Emitted[0].Data.Length); - - // Manual resume - pump.OnCapacityAvailable(); - Assert.Equal(2, target.Emitted.Count(e => !e.EndStream)); - - // Resume until complete + // Initial register starts reads. With target's 16KB chunk size and FakeTarget (no auto-resume), + // the base class credit system drains initial budget and pauses. + // We need to add credits to resume. while (target.Completed.Count == 0) { - pump.OnCapacityAvailable(); + pump.AddCredit(); } Assert.Equal(200, target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length)); @@ -141,107 +148,118 @@ public void OnCapacityAvailable_should_resume_after_capacity_exhaustion() public void Cancel_should_stop_drain() { var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 1024, maxCapacity: 2); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); var body = MakeBody(100); - pump.Register(body, 100, CancellationToken.None); - // Already completed since MemoryStream is sync with maxCapacity=2 + pump.Register(body, CancellationToken.None); + // Already completed since MemoryStream is sync with sufficient budget Assert.Single(target.Completed); // Cancel after complete should be no-op - pump.Cancel(); + pump.Cancel(0); } [Fact(Timeout = 5000)] public void Cancel_midDrain_should_not_call_onDrainComplete() { var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 64, maxCapacity: 1); - var body = MakeBody(200); - - pump.Register(body, 200, CancellationToken.None); - // maxCapacity=1, FakeTarget: 1 chunk emitted, pump paused - Assert.Single(target.Emitted, e => !e.EndStream); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); + // Large enough to not complete with 16 initial credits + var largeBody = MakeBody(1000 * 1024); + + pump.Register(largeBody, CancellationToken.None); + // Some reads completed with initial credits + var initialEmitted = target.Emitted.Count(e => !e.EndStream); + Assert.True(initialEmitted > 0, "initial credits should have started reads"); Assert.Empty(target.Completed); - pump.Cancel(); + // Cancel mid-drain + pump.Cancel(0); - // Cancel should NOT fire OnDrainComplete - Assert.Empty(target.Completed); - Assert.Empty(target.Failed); + // Cancel should NOT fire OnDrainComplete (only OnDrainFailed if read in-flight) + // With sync MemoryStream, all reads already completed, so nothing in-flight + var completedAfterCancel = target.Completed; + var failedAfterCancel = target.Failed; + Assert.Empty(completedAfterCancel); + Assert.Empty(failedAfterCancel); } [Fact(Timeout = 5000)] public void Cleanup_should_be_idempotent() { var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 1024, maxCapacity: 2); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); pump.Cleanup(); pump.Cleanup(); } [Fact(Timeout = 5000)] - public void Capacity2_should_drain_two_chunk_body_without_resume() + public void Register_should_drain_small_body_without_additional_credits() { - // maxCapacity=2 allows 2 reads before pausing. A body that fits in 2 reads - // (1 data read + 1 EOF read) completes without OnCapacityAvailable. + // Small body fits within initial budget without needing additional AddCredit calls. var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 64, maxCapacity: 2); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); var body = MakeBody(64); - pump.Register(body, 64, CancellationToken.None); + pump.Register(body, CancellationToken.None); Assert.Equal(64, target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length)); Assert.Single(target.Completed); } [Fact(Timeout = 5000)] - public void Capacity2_should_need_resume_for_large_body() + public void AddCredit_should_drain_large_body_without_limit() { - // maxCapacity=2: pump issues 2 reads, pauses, needs OnCapacityAvailable to continue. + // Very large body: initial 16 credits may not be enough depending on budget. + // Keep adding credits until drain completes. var target = new FakeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 64, maxCapacity: 2); - var body = MakeBody(200); - - pump.Register(body, 200, CancellationToken.None); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); + var largeBodySize = 1000 * 1024; // 1 MB body + var body = MakeBody(largeBodySize); - // 2 reads issued: 64 + 64 = 128 bytes emitted - Assert.Equal(128, target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length)); - Assert.Empty(target.Completed); + pump.Register(body, CancellationToken.None); - // Resume until complete - while (target.Completed.Count == 0) + // With 16 initial credits and 16 KB chunks, we drain ~256 KB. Need more credits for 1 MB. + int iterations = 0; + while (target.Completed.Count == 0 && iterations < 1000) { - pump.OnCapacityAvailable(); + pump.AddCredit(); + iterations++; } - Assert.Equal(200, target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length)); + Assert.True(iterations < 1000, "drain should complete within 1000 AddCredit calls"); + Assert.Single(target.Completed); + Assert.Equal(largeBodySize, target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length)); } [Fact(Timeout = 5000)] - public void SyncStarvationGuard_should_yield_after_max_sync_reads() + public void Sync_reads_should_complete_large_body_with_auto_resume() { - // Use AutoResumeTarget so the pump loops without manual OnCapacityAvailable calls. - // maxCapacity=1 + auto-resume = pump reads one chunk, emits, auto-resumes, reads next... - // After 64 consecutive sync reads, it yields via DrainContinue. + // Use AutoResumeTarget so the pump adds credit inline after each emit. + // Large body drains synchronously. var target = new AutoResumeTarget(); - var pump = new SerialBodyPump(target, new CancellationTokenSource(), chunkSize: 16, maxCapacity: 1); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); target.SetPump(pump); var body = MakeBody(65 * 16); - pump.Register(body, 65 * 16, CancellationToken.None); + pump.Register(body, CancellationToken.None); - // 64 data chunks emitted, then guard fired (DrainContinue to Nobody = lost) + // All 65 data chunks emitted + EOF var emittedBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); - Assert.Equal(64 * 16, emittedBytes); - Assert.Empty(target.Completed); - - // Resume — emits remaining 1 chunk + EOF - pump.HandleDrainContinue(); - - var totalBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); - Assert.Equal(65 * 16, totalBytes); + Assert.Equal(65 * 16, emittedBytes); Assert.Single(target.Completed); } @@ -249,14 +267,145 @@ public void SyncStarvationGuard_should_yield_after_max_sync_reads() public void CtsDisposal_should_happen_on_complete() { var target = new FakeTarget(); + var poolContext = new ConnectionPoolContext(); var connCts = new CancellationTokenSource(); var reqCts = new CancellationTokenSource(); - var pump = new SerialBodyPump(target, connCts, chunkSize: 1024, maxCapacity: 2); + var pump = new SerialBodyPump(target, poolContext, connCts); - pump.Register(MakeBody(100), 100, reqCts.Token); + pump.Register(MakeBody(100), reqCts.Token); Assert.Single(target.Completed); // Verify no exception when disposing reqCts (linked CTS should already be disposed by pump) reqCts.Dispose(); } + + [Fact(Timeout = 5000)] + public void PipeReader_should_drain_completed_pipe_with_auto_resume() + { + // Scenario: PipeWriter writes 64 KB in 1 KB chunks, then completes. + // PipeReader.AsStream() is registered AFTER all data is written. + // All reads should complete synchronously since data is already buffered. + // + // PauseWriterThreshold = 0 disables writer back-pressure so all FlushAsync + // calls complete synchronously without a reader consuming data first. + var pipeOptions = new PipeOptions(pauseWriterThreshold: 0, resumeWriterThreshold: 0); + var pipe = new Pipe(pipeOptions); + var totalSize = 64 * 1024; + var chunkSize = 1024; + + // Write all data first — FlushAsync completes synchronously with no back-pressure + for (var i = 0; i < totalSize / chunkSize; i++) + { + var mem = pipe.Writer.GetMemory(chunkSize); + for (var j = 0; j < chunkSize; j++) + { + mem.Span[j] = (byte)((i * chunkSize + j) % 256); + } + + pipe.Writer.Advance(chunkSize); + var flushResult = pipe.Writer.FlushAsync(); + Assert.True(flushResult.IsCompleted, "FlushAsync should complete synchronously with no back-pressure"); + } + + pipe.Writer.Complete(); + + // Now register the pump with the completed pipe + var target = new AutoResumeTarget(); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); + target.SetPump(pump); + + var bodyStream = pipe.Reader.AsStream(); + pump.Register(bodyStream, CancellationToken.None); + + var emittedBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); + Assert.Equal(totalSize, emittedBytes); + Assert.Single(target.Completed); + Assert.Empty(target.Failed); + } + + [Fact(Timeout = 5000)] + public async Task PipeReader_should_drain_pipe_when_writer_completes_after_registration() + { + // Scenario: Write a few chunks, register the pump, then write the rest. + // The first reads complete synchronously (data already buffered). + // Later reads may go async because the PipeWriter hasn't written yet. + // + // This simulates the real server handler case: the handler writes to + // PipeWriter while the pump reads from PipeReader.AsStream() concurrently. + var pipeOptions = new PipeOptions(pauseWriterThreshold: 0, resumeWriterThreshold: 0); + var pipe = new Pipe(pipeOptions); + var totalSize = 64 * 1024; + var chunkSize = 1024; + var preWriteChunks = 4; // Write 4 KB before registering + + // Write initial chunks + for (var i = 0; i < preWriteChunks; i++) + { + var mem = pipe.Writer.GetMemory(chunkSize); + for (var j = 0; j < chunkSize; j++) + { + mem.Span[j] = (byte)((i * chunkSize + j) % 256); + } + + pipe.Writer.Advance(chunkSize); + var flushResult = pipe.Writer.FlushAsync(); + Assert.True(flushResult.IsCompleted); + } + + // Register the pump BEFORE writer completes + var target = new AutoResumeTarget(); + var poolContext = new ConnectionPoolContext(); + var connCts = new CancellationTokenSource(); + var pump = new SerialBodyPump(target, poolContext, connCts); + target.SetPump(pump); + + var bodyStream = pipe.Reader.AsStream(); + pump.Register(bodyStream, CancellationToken.None); + + // At this point, the pump has consumed the initial 4 KB synchronously, + // then issued a read that went async (no more data in pipe yet). + var emittedSoFar = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); + Assert.True(emittedSoFar >= preWriteChunks * chunkSize, + $"Expected at least {preWriteChunks * chunkSize} bytes emitted, got {emittedSoFar}"); + Assert.Empty(target.Completed); // Not done yet — writer hasn't completed + + // Now write the remaining chunks from a background task. + // The async reads dispatched via PipeTo need an actor to receive the messages. + // Since we don't have an actor system in this unit test, the PipeTo target is + // ActorRefs.Nobody — async reads will be lost. + // + // This proves the core issue: when PipeReader.AsStream().ReadAsync() goes async, + // the SerialBodyPump dispatches via PipeTo to ActorRefs.Nobody, and the drain stalls. + await Task.Run(async () => + { + for (var i = preWriteChunks; i < totalSize / chunkSize; i++) + { + var mem = pipe.Writer.GetMemory(chunkSize); + for (var j = 0; j < chunkSize; j++) + { + mem.Span[j] = (byte)((i * chunkSize + j) % 256); + } + + pipe.Writer.Advance(chunkSize); + await pipe.Writer.FlushAsync(); + } + + pipe.Writer.Complete(); + }); + + // Give a short window for any async completions to arrive + await Task.Delay(100); + + // The pump is stalled — no actor receives the PipeTo messages. + // With a real actor system, HandleReadComplete would be called, but here + // the drain should NOT have completed because PipeTo goes to Nobody. + var finalBytes = target.Emitted.Where(e => !e.EndStream).Sum(e => e.Data.Length); + Assert.True(finalBytes < totalSize, + $"Expected drain to stall (async reads lost to Nobody), but got {finalBytes}/{totalSize} bytes. " + + "If this unexpectedly passes, PipeReader.AsStream() may be completing reads synchronously " + + "even when data arrives after the read was issued."); + Assert.Empty(target.Completed); + } } diff --git a/src/TurboHTTP.Tests/Protocol/Syntax/Http2/FlowControllerReservationSpec.cs b/src/TurboHTTP.Tests/Protocol/Syntax/Http2/FlowControllerReservationSpec.cs new file mode 100644 index 000000000..ae80d6c26 --- /dev/null +++ b/src/TurboHTTP.Tests/Protocol/Syntax/Http2/FlowControllerReservationSpec.cs @@ -0,0 +1,66 @@ +using TurboHTTP.Protocol.Syntax.Http2; + +namespace TurboHTTP.Tests.Protocol.Syntax.Http2; + +public sealed class FlowControllerReservationSpec +{ + private static FlowController CreateController(int connectionWindow = 64 * 1024, int streamWindow = 64 * 1024) + { + // Initial send windows are always 65535 per RFC 9113, so we initialize with + // larger recv windows and let the constructor set send to 65535 + var fc = new FlowController( + connectionWindowSize: connectionWindow, + streamWindowSize: streamWindow); + return fc; + } + + [Fact(Timeout = 5000)] + public void Reserve_should_decrement_both_windows() + { + var fc = CreateController(connectionWindow: 64 * 1024, streamWindow: 32 * 1024); + fc.InitStreamSendWindow(1); + + fc.Reserve(1, 16 * 1024); + + Assert.Equal(65535 - 16 * 1024, fc.ConnectionSendWindow); + Assert.Equal(65535 - 16 * 1024, fc.GetStreamSendWindow(1)); + } + + [Fact(Timeout = 5000)] + public void Refund_should_increment_both_windows() + { + var fc = CreateController(connectionWindow: 64 * 1024, streamWindow: 32 * 1024); + fc.InitStreamSendWindow(1); + fc.Reserve(1, 16 * 1024); + + fc.Refund(1, 4 * 1024); + + Assert.Equal(65535 - 12 * 1024, fc.ConnectionSendWindow); + Assert.Equal(65535 - 12 * 1024, fc.GetStreamSendWindow(1)); + } + + [Fact(Timeout = 5000)] + public void Reserve_should_reflect_outstanding_reservations_in_connection_window() + { + var fc = CreateController(connectionWindow: 32 * 1024, streamWindow: 32 * 1024); + fc.InitStreamSendWindow(1); + fc.InitStreamSendWindow(3); + + fc.Reserve(1, 16 * 1024); + fc.Reserve(3, 16 * 1024); + + Assert.Equal(65535 - 32 * 1024, fc.ConnectionSendWindow); + } + + [Fact(Timeout = 5000)] + public void Refund_with_zero_should_be_noop() + { + var fc = CreateController(connectionWindow: 64 * 1024, streamWindow: 32 * 1024); + fc.InitStreamSendWindow(1); + fc.Reserve(1, 16 * 1024); + + fc.Refund(1, 0); + + Assert.Equal(65535 - 16 * 1024, fc.ConnectionSendWindow); + } +} diff --git a/src/TurboHTTP/Protocol/Body/BodyDrainSlot.cs b/src/TurboHTTP/Protocol/Body/BodyDrainSlot.cs index 163499a36..aaa5474b6 100644 --- a/src/TurboHTTP/Protocol/Body/BodyDrainSlot.cs +++ b/src/TurboHTTP/Protocol/Body/BodyDrainSlot.cs @@ -8,7 +8,6 @@ internal sealed class BodyDrainSlot : IResettable // Identity — set once via Initialize, read-only after public TStreamId StreamId { get; private set; } = default!; public Stream? BodyStream { get; private set; } - public long? ContentLength { get; private set; } public CancellationToken RequestCt { get; private set; } public CancellationTokenSource? LinkedCts { get; private set; } @@ -19,24 +18,21 @@ internal sealed class BodyDrainSlot : IResettable // Managed resources public IMemoryOwner? Buffer { get; private set; } + // H2 flow-control: reserved send window for in-flight read + public int ReservedWindow { get; set; } + // Observable state public bool IsReadInFlight { get; private set; } public bool IsOrphaned { get; private set; } - public bool HasLimbo { get; private set; } - public bool IsDrainComplete { get; private set; } - public ReadOnlyMemory LimboData { get; private set; } - public int ConsecutiveSyncReads { get; private set; } public void Initialize( TStreamId streamId, Stream bodyStream, - long? contentLength, CancellationToken requestCt, CancellationTokenSource? linkedCts) { StreamId = streamId; BodyStream = bodyStream; - ContentLength = contentLength; RequestCt = requestCt; LinkedCts = linkedCts; BuildTransforms(); @@ -56,47 +52,16 @@ public void EnsureBuffer(int chunkSize) public void BeginRead() => IsReadInFlight = true; - public void CompleteSyncRead() => IsReadInFlight = false; - - public void CompleteAsyncRead() - { - IsReadInFlight = false; - ConsecutiveSyncReads = 0; - } - - /// - /// Increments the consecutive sync read counter. - /// Returns true when the starvation threshold is reached (caller should yield). - /// - public bool IncrementSyncReads(int maxPerDispatch) - { - ConsecutiveSyncReads++; - return ConsecutiveSyncReads >= maxPerDispatch; - } - - public void ResetSyncReads() => ConsecutiveSyncReads = 0; + public void CompleteRead() => IsReadInFlight = false; public void MarkOrphaned() => IsOrphaned = true; - public void StoreLimbo(ReadOnlyMemory data) + public void ReplaceBuffer(int newSize) { - LimboData = data; - HasLimbo = true; - } - - public void ShrinkLimbo(int consumed) - { - LimboData = LimboData[consumed..]; - } - - public void ClearLimbo() - { - LimboData = default; - HasLimbo = false; + Buffer?.Dispose(); + Buffer = MemoryPool.Shared.Rent(newSize); } - public void MarkDrainComplete() => IsDrainComplete = true; - public void DisposeResources() { Buffer?.Dispose(); @@ -112,13 +77,9 @@ public void Reset() Buffer = null; LinkedCts = null; RequestCt = default; - ContentLength = null; + ReservedWindow = 0; IsReadInFlight = false; IsOrphaned = false; - LimboData = default; - HasLimbo = false; - IsDrainComplete = false; - ConsecutiveSyncReads = 0; CachedSuccessTransform = null; CachedFailureTransform = null; } diff --git a/src/TurboHTTP/Protocol/Body/BodyPumpBase.cs b/src/TurboHTTP/Protocol/Body/BodyPumpBase.cs new file mode 100644 index 000000000..a6addf3cf --- /dev/null +++ b/src/TurboHTTP/Protocol/Body/BodyPumpBase.cs @@ -0,0 +1,336 @@ +using Akka.Actor; +using TurboHTTP.Pooling; + +namespace TurboHTTP.Protocol.Body; + +internal abstract class BodyPumpBase where TStreamId : notnull +{ + private const int MinBudget = 8; + private const int MaxBudget = 48; + private const double Alpha = 0.3; + private const long SlowThresholdTicks = TimeSpan.TicksPerMillisecond * 10; + private const long FastThresholdTicks = TimeSpan.TicksPerMillisecond / 2; + + private readonly IBodyDrainTarget _target; + private readonly ConnectionPoolContext _poolContext; + private readonly CancellationTokenSource _connectionCts; + + private readonly Queue _readyQueue = new(); + private readonly Dictionary> _activeSlots = new(); + private readonly HashSet _cancelledStreams = new(); + + private int _credits; + private int _budget; + private double _ema; + private long _lastPullTicks; + + protected BodyPumpBase( + IBodyDrainTarget target, + ConnectionPoolContext poolContext, + CancellationTokenSource connectionCts) + { + _target = target; + _poolContext = poolContext; + _connectionCts = connectionCts; + _ema = (SlowThresholdTicks + FastThresholdTicks) / 2.0; + _budget = MapBudget(_ema); + _lastPullTicks = Environment.TickCount64 * TimeSpan.TicksPerMillisecond; + } + + public void AddCredit() + { + UpdateEma(); + _credits = Math.Min(_credits + 1, MaxBudget); + + var threshold = Math.Max(Math.Min(_budget / 2, _activeSlots.Count), 1); + if (_credits >= threshold) + { + DrainReady(_budget); + } + else if (_credits > 0 && _readyQueue.Count > 0) + { + DrainReady(1); + } + } + + public void Register(TStreamId streamId, Stream bodyStream, CancellationToken requestCt, int initialCredits = 0) + { + var slot = _poolContext.Rent(static () => new BodyDrainSlot()); + var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(requestCt, _connectionCts.Token); + slot.Initialize(streamId, bodyStream, requestCt, linkedCts); + slot.EnsureBuffer(_target.PreferredChunkSize); + _activeSlots[streamId] = slot; + EnqueueStream(streamId); + + if (_target.HasPendingDemand) + { + _credits++; + DrainReady(1); + } + + for (var i = 0; i < initialCredits; i++) + { + AddCredit(); + } + } + + public void HandleReadComplete(TStreamId streamId, int bytesRead) + { + if (!_activeSlots.TryGetValue(streamId, out var slot)) + { + return; + } + + slot.CompleteRead(); + + if (slot.IsOrphaned) + { + AfterRead(streamId, slot, 0); + CleanupSlot(streamId, slot); + return; + } + + ProcessReadResult(streamId, slot, bytesRead); + + if (_credits > 0) + { + DrainReady(1); + } + } + + public void HandleReadFailed(TStreamId streamId, Exception reason) + { + if (!_activeSlots.TryGetValue(streamId, out var slot)) + { + return; + } + + slot.CompleteRead(); + + if (slot.IsOrphaned) + { + AfterRead(streamId, slot, 0); + CleanupSlot(streamId, slot); + return; + } + + AfterRead(streamId, slot, 0); + + _target.OnDrainFailed(streamId, reason); + CleanupSlot(streamId, slot); + } + + public void Cancel(TStreamId streamId) + { + if (!_activeSlots.TryGetValue(streamId, out var slot)) + { + return; + } + + if (slot.IsReadInFlight) + { + slot.MarkOrphaned(); + slot.LinkedCts?.Cancel(); + _cancelledStreams.Add(streamId); + } + else + { + CleanupSlot(streamId, slot); + } + + OnStreamCancelled(streamId); + } + + public void CancelAll() + { + _connectionCts.Cancel(); + + foreach (var (_, slot) in _activeSlots) + { + if (slot.IsReadInFlight) + { + slot.MarkOrphaned(); + } + else + { + DisposeSlot(slot); + } + } + + _activeSlots.Clear(); + _readyQueue.Clear(); + _cancelledStreams.Clear(); + _credits = 0; + OnCancelAll(); + } + + // H2 flow-control extension points — only FlowControlledBodyPump overrides these + protected virtual void OnCancelAll() { } + + protected virtual void OnStreamCancelled(TStreamId streamId) { } + + // When returning false, the override is responsible for tracking the stream + // (e.g., moving it to a blocked set). The stream is removed from the ready queue. + protected virtual bool IsStreamEligible(TStreamId streamId, BodyDrainSlot slot) => true; + + protected virtual int ComputeReadSize(TStreamId streamId, BodyDrainSlot slot) + => _target.PreferredChunkSize; + + protected virtual void BeforeRead(TStreamId streamId, BodyDrainSlot slot) { } + + protected virtual void AfterRead(TStreamId streamId, BodyDrainSlot slot, int bytesRead) { } + + protected virtual void EnqueueStream(TStreamId streamId) + => _readyQueue.Enqueue(streamId); + + private void DrainReady(int maxReads) + { + var reads = 0; + var queueSize = _readyQueue.Count; + while (reads < maxReads && _credits > 0 && queueSize-- > 0) + { + if (!_readyQueue.TryDequeue(out var streamId)) + { + break; + } + + if (_cancelledStreams.Remove(streamId)) + { + continue; + } + + if (!_activeSlots.TryGetValue(streamId, out var slot)) + { + continue; + } + + if (slot.IsReadInFlight) + { + _readyQueue.Enqueue(streamId); + continue; + } + + if (!IsStreamEligible(streamId, slot)) + { + continue; + } + + PerformRead(streamId, slot); + reads++; + } + } + + private void PerformRead(TStreamId streamId, BodyDrainSlot slot) + { + if (slot.Buffer!.Memory.Length < _target.PreferredChunkSize) + { + slot.ReplaceBuffer(_target.PreferredChunkSize); + } + + var readSize = ComputeReadSize(streamId, slot); + BeforeRead(streamId, slot); + var result = StartRead(slot, readSize, _target.PipeToTarget); + _credits--; + + if (result.Outcome == ReadOutcome.CompletedSynchronously) + { + ProcessReadResult(streamId, slot, result.BytesRead); + } + } + + private void ProcessReadResult(TStreamId streamId, BodyDrainSlot slot, int bytesRead) + { + AfterRead(streamId, slot, bytesRead); + + if (bytesRead == 0) + { + _target.EmitDataFrames(streamId, default, endStream: true); + // Clean up the slot BEFORE notifying the target. OnDrainComplete triggers + // CloseStream which calls _pump.Cancel(streamId). If the slot is still in + // _activeSlots, Cancel will CleanupSlot a second time, double-returning it + // to the pool. A later rent then shares the slot with another stream, and + // the stale cleanup nulls its Buffer, causing a NullReferenceException. + CleanupSlot(streamId, slot); + _target.OnDrainComplete(streamId); + return; + } + + _readyQueue.Enqueue(streamId); + _target.EmitDataFrames(streamId, slot.Buffer!.Memory[..bytesRead], endStream: false); + } + + private void UpdateEma() + { + var nowTicks = Environment.TickCount64 * TimeSpan.TicksPerMillisecond; + var interval = nowTicks - _lastPullTicks; + _ema = Alpha * interval + (1 - Alpha) * _ema; + _budget = MapBudget(_ema); + _lastPullTicks = nowTicks; + } + + private static int MapBudget(double ema) + { + if (ema <= FastThresholdTicks) + { + return MaxBudget; + } + + if (ema >= SlowThresholdTicks) + { + return MinBudget; + } + + var ratio = (ema - FastThresholdTicks) / (SlowThresholdTicks - FastThresholdTicks); + return MaxBudget - (int)(ratio * (MaxBudget - MinBudget)); + } + + private static ReadResult StartRead( + BodyDrainSlot slot, + int chunkSize, + IActorRef pipeToTarget) + { + slot.BeginRead(); + var token = slot.LinkedCts?.Token ?? slot.RequestCt; + var vt = slot.BodyStream!.ReadAsync(slot.Buffer!.Memory[..chunkSize], token); + + if (vt.IsCompletedSuccessfully) + { + slot.CompleteRead(); + return new ReadResult(ReadOutcome.CompletedSynchronously, vt.Result); + } + + vt.PipeTo( + pipeToTarget, + success: slot.CachedSuccessTransform, + failure: slot.CachedFailureTransform); + return new ReadResult(ReadOutcome.Dispatched, 0); + } + + private void CleanupSlot(TStreamId streamId, BodyDrainSlot slot) + { + _activeSlots.Remove(streamId); + DisposeSlot(slot); + } + + private void DisposeSlot(BodyDrainSlot slot) + { + slot.DisposeResources(); + slot.Reset(); + _poolContext.Return(slot); + } + + private readonly record struct ReadResult(ReadOutcome Outcome, int BytesRead); + + private enum ReadOutcome + { + CompletedSynchronously, + Dispatched + } + + // Protected accessors for subclasses and testing + protected int GetCredits() => _credits; + protected int GetBudget() => _budget; + protected int GetActiveStreamCount() => _activeSlots.Count; + protected ConnectionPoolContext PoolContext => _poolContext; + protected IBodyDrainTarget Target => _target; +} diff --git a/src/TurboHTTP/Protocol/Body/BodyPumpHelper.cs b/src/TurboHTTP/Protocol/Body/BodyPumpHelper.cs deleted file mode 100644 index b79d57426..000000000 --- a/src/TurboHTTP/Protocol/Body/BodyPumpHelper.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Akka.Actor; - -namespace TurboHTTP.Protocol.Body; - -internal static class BodyPumpHelper -{ - public const int MaxSyncReadsPerDispatch = 64; - - /// - /// Issues one read against 's body stream. - /// - /// — data is in ; caller processes inline. - /// — async read was PipeTo'd; caller waits for or . - /// - /// Caller is responsible for starvation guard (checking - /// against before calling this method). - /// - public static ReadResult StartRead( - BodyDrainSlot slot, - int chunkSize, - IActorRef stageActor) - { - slot.BeginRead(); - var token = slot.LinkedCts?.Token ?? slot.RequestCt; - var vt = slot.BodyStream!.ReadAsync(slot.Buffer!.Memory[..chunkSize], token); - - if (vt.IsCompletedSuccessfully) - { - slot.CompleteSyncRead(); - slot.IncrementSyncReads(MaxSyncReadsPerDispatch); - return new ReadResult(ReadOutcome.CompletedSynchronously, vt.Result); - } - - vt.PipeTo( - stageActor, - success: slot.CachedSuccessTransform, - failure: slot.CachedFailureTransform); - return new ReadResult(ReadOutcome.Dispatched, 0); - } - - internal readonly record struct ReadResult(ReadOutcome Outcome, int BytesRead); - - internal enum ReadOutcome - { - CompletedSynchronously, - Dispatched - } -} diff --git a/src/TurboHTTP/Protocol/Body/DrainMessages.cs b/src/TurboHTTP/Protocol/Body/DrainMessages.cs index 0eb298dcd..5faf88e15 100644 --- a/src/TurboHTTP/Protocol/Body/DrainMessages.cs +++ b/src/TurboHTTP/Protocol/Body/DrainMessages.cs @@ -2,4 +2,3 @@ namespace TurboHTTP.Protocol.Body; internal readonly record struct DrainReadComplete(TStreamId StreamId, int BytesRead); internal readonly record struct DrainReadFailed(TStreamId StreamId, Exception Reason); -internal readonly record struct DrainContinue(TStreamId StreamId); diff --git a/src/TurboHTTP/Protocol/Body/FlowControlledBodyPump.cs b/src/TurboHTTP/Protocol/Body/FlowControlledBodyPump.cs index ec49dcce8..b889b8c98 100644 --- a/src/TurboHTTP/Protocol/Body/FlowControlledBodyPump.cs +++ b/src/TurboHTTP/Protocol/Body/FlowControlledBodyPump.cs @@ -1,409 +1,122 @@ -using Akka.Actor; using TurboHTTP.Pooling; using TurboHTTP.Protocol.Syntax.Http2; namespace TurboHTTP.Protocol.Body; -internal sealed class FlowControlledBodyPump +internal sealed class FlowControlledBodyPump : BodyPumpBase { - private readonly IBodyDrainTarget _target; private readonly FlowController _flowController; - private readonly ConnectionPoolContext _poolContext; - private readonly CancellationTokenSource _connectionCts; - private readonly int _chunkSize; - private readonly int _hardCap; - - private readonly Queue _readyQueue = new(); - private readonly Dictionary> _activeSlots = new(); - private readonly HashSet _cancelledStreams = new(); private readonly HashSet _windowBlockedStreams = new(); - private readonly HashSet _limboSlots = new(); - - private int _readSlots = 2; - private int _asyncInFlight; + private readonly List _unblockedTemp = new(); public FlowControlledBodyPump( IBodyDrainTarget target, FlowController flowController, ConnectionPoolContext poolContext, - CancellationTokenSource connectionCts, - int chunkSize, - int hardCap) + CancellationTokenSource connectionCts) + : base(target, poolContext, connectionCts) { - _target = target; _flowController = flowController; - _poolContext = poolContext; - _connectionCts = connectionCts; - _chunkSize = chunkSize; - _hardCap = hardCap; - } - - public void Register(int streamId, Stream bodyStream, long? contentLength, CancellationToken requestCt) - { - var slot = _poolContext.Rent(static () => new BodyDrainSlot()); - var linkedCts = requestCt.CanBeCanceled - ? CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token, requestCt) - : CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token); - slot.Initialize(streamId, bodyStream, contentLength, requestCt, linkedCts); - slot.EnsureBuffer(_chunkSize); - _activeSlots[streamId] = slot; - - if (_flowController.GetStreamSendWindow(streamId) > 0) - { - _readyQueue.Enqueue(streamId); - TryScheduleReads(); - } - else - { - _windowBlockedStreams.Add(streamId); - } - } - - public void RegisterWithLimbo(int streamId, ReadOnlyMemory data, CancellationToken requestCt) - { - var slot = _poolContext.Rent(static () => new BodyDrainSlot()); - var linkedCts = requestCt.CanBeCanceled - ? CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token, requestCt) - : CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token); - slot.Initialize(streamId, null!, null, requestCt, linkedCts); - - // EnsureBuffer rents from MemoryPool (slot is fresh after pool return, Buffer is null). - // Copy remainder data into the slot's buffer so limbo slice points into owned memory. - slot.EnsureBuffer(Math.Max(data.Length, _chunkSize)); - data.CopyTo(slot.Buffer!.Memory); - slot.StoreLimbo(slot.Buffer.Memory[..data.Length]); - - // No body stream — limbo is the entire remaining payload; mark as complete so - // DrainLimboSlot emits END_STREAM and calls CompleteDrain after the flush. - slot.MarkDrainComplete(); - - _activeSlots[streamId] = slot; - _limboSlots.Add(streamId); } public void OnWindowUpdate(int streamId) { if (streamId == 0) { - // Connection-level: drain all limbo slots, unblock all window-blocked streams - var limboSnapshot = _limboSlots.ToArray(); - foreach (var id in limboSnapshot) + // Connection-level update: re-evaluate all blocked streams and unblock eligible ones. + _unblockedTemp.Clear(); + foreach (var blocked in _windowBlockedStreams) { - if (_activeSlots.TryGetValue(id, out var slot)) + var window = Math.Min( + _flowController.GetStreamSendWindow(blocked), + _flowController.ConnectionSendWindow); + if (window >= ComputeMinReadSize()) { - DrainLimboSlot(slot); + _unblockedTemp.Add(blocked); } } - foreach (var id in _windowBlockedStreams) + foreach (var id in _unblockedTemp) { - _readyQueue.Enqueue(id); + _windowBlockedStreams.Remove(id); + EnqueueStream(id); } - - _windowBlockedStreams.Clear(); } - else + else if (_windowBlockedStreams.Remove(streamId)) { - if (_windowBlockedStreams.Remove(streamId)) - { - _readyQueue.Enqueue(streamId); - } - else if (_limboSlots.Contains(streamId)) - { - if (_activeSlots.TryGetValue(streamId, out var slot)) - { - DrainLimboSlot(slot); - } - } + EnqueueStream(streamId); } - TryScheduleReads(); - } - - public void HandleReadComplete(int streamId, int bytesRead) - { - _asyncInFlight--; - - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.CompleteAsyncRead(); - - if (slot.IsOrphaned) + // Always inject credits when active streams exist. The pump can reach zero credits + // legitimately: the initial burst consumes bootstrap credits, all streams become + // window-blocked, and no further OnOutboundFlushed calls replenish credits because + // no data is being pushed. When a WINDOW_UPDATE subsequently unblocks streams (or + // streams are already in the ready queue from a prior re-enqueue), the pump must be + // able to read them. Without this unconditional boost the pump deadlocks — streams + // sit in the ready queue with available window but zero credits to drive reads, + // eventually tripping the data-rate monitor which RST_STREAMs the connection. + if (GetActiveStreamCount() > 0) { - RemoveAndReturnSlot(streamId, slot); - return; - } - - ProcessReadResult(slot, bytesRead); - } - - public void HandleReadFailed(int streamId, Exception reason) - { - _asyncInFlight--; - - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.CompleteAsyncRead(); - - if (slot.IsOrphaned) - { - RemoveAndReturnSlot(streamId, slot); - return; + for (var i = 0; i < 16; i++) + { + AddCredit(); + } } - - _limboSlots.Remove(streamId); - _activeSlots.Remove(streamId); - _target.OnDrainFailed(streamId, reason); - slot.DisposeResources(); - _poolContext.Return(slot); } - public void HandleDrainContinue(int streamId) - { - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.CompleteSyncRead(); + public void Cleanup() => CancelAll(); - if (slot.IsOrphaned) - { - RemoveAndReturnSlot(streamId, slot); - return; - } - - _readyQueue.Enqueue(streamId); - TryScheduleReads(); - } - - public void Cancel(int streamId) + protected override bool IsStreamEligible(int streamId, BodyDrainSlot slot) { - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.LinkedCts?.Cancel(); - - if (slot.IsReadInFlight) - { - slot.MarkOrphaned(); - return; - } - - if (_limboSlots.Remove(streamId)) - { - RemoveAndReturnSlot(streamId, slot); - return; - } + var available = Math.Min( + _flowController.GetStreamSendWindow(streamId), + _flowController.ConnectionSendWindow); - if (_windowBlockedStreams.Remove(streamId)) + if (available < ComputeMinReadSize()) { - RemoveAndReturnSlot(streamId, slot); - return; + _windowBlockedStreams.Add(streamId); + return false; } - // In ready queue: use lazy removal via _cancelledStreams - _cancelledStreams.Add(streamId); + return true; } - public void Cleanup() + protected override int ComputeReadSize(int streamId, BodyDrainSlot slot) { - _connectionCts.Cancel(); - - foreach (var (streamId, slot) in _activeSlots) - { - if (!slot.IsOrphaned) - { - slot.DisposeResources(); - _poolContext.Return(slot); - } - - _limboSlots.Remove(streamId); - } - - _activeSlots.Clear(); - _windowBlockedStreams.Clear(); - _cancelledStreams.Clear(); - _readyQueue.Clear(); + var chunkSize = Target.PreferredChunkSize; + var available = Math.Min( + _flowController.GetStreamSendWindow(streamId), + _flowController.ConnectionSendWindow); + return (int)Math.Min(chunkSize, available); } - private void TryScheduleReads() + protected override void BeforeRead(int streamId, BodyDrainSlot slot) { - var connWindow = _flowController.ConnectionSendWindow; - - if (connWindow <= 0) - { - return; - } - - var windowSlots = connWindow >= _chunkSize - ? (int)Math.Min(connWindow / _chunkSize, _hardCap) - : 1; - var effectiveSlots = Math.Min(Math.Min(_readSlots, windowSlots), _hardCap); - - while (_asyncInFlight < effectiveSlots && _readyQueue.Count > 0) - { - var streamId = _readyQueue.Dequeue(); - - if (_cancelledStreams.Remove(streamId)) - { - if (_activeSlots.TryGetValue(streamId, out var cancelled)) - { - RemoveAndReturnSlot(streamId, cancelled); - } - - continue; - } - - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - continue; - } - - if (_flowController.GetStreamSendWindow(streamId) <= 0) - { - _windowBlockedStreams.Add(streamId); - continue; - } - - // Starvation guard: yield after MaxSyncReadsPerDispatch consecutive sync reads - // so the actor thread can process other messages between bursts. - if (slot.ConsecutiveSyncReads >= BodyPumpHelper.MaxSyncReadsPerDispatch) - { - slot.ResetSyncReads(); - slot.BeginRead(); - _target.StageActor.Tell(new DrainContinue(slot.StreamId), ActorRefs.NoSender); - continue; - } - - var result = BodyPumpHelper.StartRead(slot, _chunkSize, _target.StageActor); - - switch (result.Outcome) - { - case BodyPumpHelper.ReadOutcome.CompletedSynchronously: - ProcessReadResult(slot, result.BytesRead); - break; - - case BodyPumpHelper.ReadOutcome.Dispatched: - _asyncInFlight++; - break; - } - } + var reserve = ComputeReadSize(streamId, slot); + _flowController.Reserve(streamId, reserve); + slot.ReservedWindow = reserve; } - private void ProcessReadResult(BodyDrainSlot slot, int bytesRead) + protected override void AfterRead(int streamId, BodyDrainSlot slot, int bytesRead) { - if (bytesRead == 0) + var refund = slot.ReservedWindow - bytesRead; + if (refund > 0) { - if (slot.HasLimbo) - { - slot.MarkDrainComplete(); - DrainLimboSlot(slot); - } - else - { - _target.EmitDataFrames(slot.StreamId, default, endStream: true); - CompleteDrain(slot); - } - - return; + _flowController.Refund(streamId, refund); } - var streamWindow = _flowController.GetStreamSendWindow(slot.StreamId); - var connWindow = _flowController.ConnectionSendWindow; - var window = (int)Math.Min(streamWindow, connWindow); - var data = slot.Buffer!.Memory[..bytesRead]; - - if (window >= bytesRead) - { - _target.EmitDataFrames(slot.StreamId, data, endStream: false); - _flowController.OnDataSent(slot.StreamId, bytesRead); - _readSlots = Math.Min(_readSlots + 1, _hardCap); - _readyQueue.Enqueue(slot.StreamId); - TryScheduleReads(); - } - else if (window > 0) - { - _target.EmitDataFrames(slot.StreamId, data[..window], endStream: false); - _flowController.OnDataSent(slot.StreamId, window); - slot.StoreLimbo(data[window..]); - _limboSlots.Add(slot.StreamId); - - if (connWindow <= window) - { - _readSlots = Math.Max(_readSlots / 2, 1); - } - } - else - { - slot.StoreLimbo(data); - _limboSlots.Add(slot.StreamId); - - if (connWindow == 0) - { - _readSlots = Math.Max(_readSlots / 2, 1); - } - } + slot.ReservedWindow = 0; } - private void DrainLimboSlot(BodyDrainSlot slot) + protected override void OnCancelAll() { - var connWindow = _flowController.ConnectionSendWindow; - var streamWindow = slot.BodyStream is null - ? connWindow - : _flowController.GetStreamSendWindow(slot.StreamId); - var window = (int)Math.Min(streamWindow, connWindow); - - if (window <= 0) - { - return; - } - - if (window >= slot.LimboData.Length) - { - _target.EmitDataFrames(slot.StreamId, slot.LimboData, endStream: slot.IsDrainComplete); - _flowController.OnDataSent(slot.StreamId, slot.LimboData.Length); - _limboSlots.Remove(slot.StreamId); - slot.ClearLimbo(); - - if (slot.IsDrainComplete) - { - CompleteDrain(slot); - } - else - { - _readyQueue.Enqueue(slot.StreamId); - } - } - else - { - _target.EmitDataFrames(slot.StreamId, slot.LimboData[..window], endStream: false); - _flowController.OnDataSent(slot.StreamId, window); - slot.ShrinkLimbo(window); - } + _windowBlockedStreams.Clear(); } - private void CompleteDrain(BodyDrainSlot slot) + protected override void OnStreamCancelled(int streamId) { - _limboSlots.Remove(slot.StreamId); - _activeSlots.Remove(slot.StreamId); - _target.OnDrainComplete(slot.StreamId); - slot.DisposeResources(); - _poolContext.Return(slot); + _windowBlockedStreams.Remove(streamId); } - private void RemoveAndReturnSlot(int streamId, BodyDrainSlot slot) - { - _activeSlots.Remove(streamId); - slot.DisposeResources(); - _poolContext.Return(slot); - } + private int ComputeMinReadSize() => Target.PreferredChunkSize / 2; } diff --git a/src/TurboHTTP/Protocol/Body/IBodyDrainTarget.cs b/src/TurboHTTP/Protocol/Body/IBodyDrainTarget.cs index ad8fe2770..58ffb6fce 100644 --- a/src/TurboHTTP/Protocol/Body/IBodyDrainTarget.cs +++ b/src/TurboHTTP/Protocol/Body/IBodyDrainTarget.cs @@ -4,7 +4,9 @@ namespace TurboHTTP.Protocol.Body; internal interface IBodyDrainTarget { - IActorRef StageActor { get; } + IActorRef PipeToTarget { get; } + bool HasPendingDemand { get; } + int PreferredChunkSize { get; } void EmitDataFrames(TStreamId streamId, ReadOnlyMemory data, bool endStream); void OnDrainComplete(TStreamId streamId); void OnDrainFailed(TStreamId streamId, Exception reason); diff --git a/src/TurboHTTP/Protocol/Body/MultiplexedBodyPump.cs b/src/TurboHTTP/Protocol/Body/MultiplexedBodyPump.cs index aecb92dad..944d7e6f8 100644 --- a/src/TurboHTTP/Protocol/Body/MultiplexedBodyPump.cs +++ b/src/TurboHTTP/Protocol/Body/MultiplexedBodyPump.cs @@ -1,208 +1,16 @@ -using Akka.Actor; using TurboHTTP.Pooling; namespace TurboHTTP.Protocol.Body; -internal sealed class MultiplexedBodyPump +internal sealed class MultiplexedBodyPump : BodyPumpBase { - private readonly IBodyDrainTarget _target; - private readonly ConnectionPoolContext _poolContext; - private readonly CancellationTokenSource _connectionCts; - private readonly int _chunkSize; - private readonly int _maxConcurrentReads; - - private readonly Queue _readyQueue = new(); - private readonly Dictionary> _activeSlots = new(); - - private int _asyncInFlight; - public MultiplexedBodyPump( IBodyDrainTarget target, ConnectionPoolContext poolContext, - CancellationTokenSource connectionCts, - int chunkSize, - int maxConcurrentReads = 4) - { - _target = target; - _poolContext = poolContext; - _connectionCts = connectionCts; - _chunkSize = chunkSize; - _maxConcurrentReads = maxConcurrentReads; - } - - public void Register(long streamId, Stream bodyStream, long? contentLength, CancellationToken requestCt) - { - var slot = _poolContext.Rent(static () => new BodyDrainSlot()); - CancellationTokenSource? linkedCts = requestCt.CanBeCanceled - ? CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token, requestCt) - : null; - var effectiveToken = linkedCts?.Token ?? _connectionCts.Token; - slot.Initialize(streamId, bodyStream, contentLength, effectiveToken, linkedCts); - slot.EnsureBuffer(_chunkSize); - _activeSlots[streamId] = slot; - _readyQueue.Enqueue(streamId); - TryScheduleReads(); - } - - public void HandleReadComplete(long streamId, int bytesRead) - { - _asyncInFlight--; - - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.CompleteAsyncRead(); - - if (slot.IsOrphaned) - { - RemoveAndReturnSlot(streamId, slot); - return; - } - - ProcessReadResult(slot, bytesRead); - } - - public void HandleReadFailed(long streamId, Exception reason) - { - _asyncInFlight--; - - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.CompleteAsyncRead(); - - if (slot.IsOrphaned) - { - RemoveAndReturnSlot(streamId, slot); - return; - } - - _activeSlots.Remove(streamId); - _target.OnDrainFailed(streamId, reason); - slot.DisposeResources(); - _poolContext.Return(slot); - } - - public void HandleDrainContinue(long streamId) - { - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.CompleteSyncRead(); - - if (slot.IsOrphaned) - { - RemoveAndReturnSlot(streamId, slot); - return; - } - - _readyQueue.Enqueue(streamId); - TryScheduleReads(); - } - - public void Cancel(long streamId) - { - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - return; - } - - slot.LinkedCts?.Cancel(); - - if (slot.IsReadInFlight) - { - slot.MarkOrphaned(); - return; - } - - RemoveAndReturnSlot(streamId, slot); - } - - public void Cleanup() + CancellationTokenSource connectionCts) + : base(target, poolContext, connectionCts) { - _connectionCts.Cancel(); - - foreach (var (_, slot) in _activeSlots) - { - if (!slot.IsOrphaned) - { - slot.DisposeResources(); - _poolContext.Return(slot); - } - } - - _activeSlots.Clear(); - _readyQueue.Clear(); } - private void TryScheduleReads() - { - while (_asyncInFlight < _maxConcurrentReads && _readyQueue.Count > 0) - { - var streamId = _readyQueue.Dequeue(); - - if (!_activeSlots.TryGetValue(streamId, out var slot)) - { - continue; - } - - // Starvation guard: yield after MaxSyncReadsPerDispatch consecutive sync reads - // so the actor thread can process other messages between bursts. - if (slot.ConsecutiveSyncReads >= BodyPumpHelper.MaxSyncReadsPerDispatch) - { - slot.ResetSyncReads(); - slot.BeginRead(); - _target.StageActor.Tell(new DrainContinue(slot.StreamId), ActorRefs.NoSender); - continue; - } - - var result = BodyPumpHelper.StartRead(slot, _chunkSize, _target.StageActor); - - switch (result.Outcome) - { - case BodyPumpHelper.ReadOutcome.CompletedSynchronously: - ProcessReadResult(slot, result.BytesRead); - break; - - case BodyPumpHelper.ReadOutcome.Dispatched: - _asyncInFlight++; - break; - } - } - } - - private void ProcessReadResult(BodyDrainSlot slot, int bytesRead) - { - if (bytesRead == 0) - { - _target.EmitDataFrames(slot.StreamId, default, endStream: true); - CompleteDrain(slot); - return; - } - - _target.EmitDataFrames(slot.StreamId, slot.Buffer!.Memory[..bytesRead], endStream: false); - _readyQueue.Enqueue(slot.StreamId); - TryScheduleReads(); - } - - private void CompleteDrain(BodyDrainSlot slot) - { - _activeSlots.Remove(slot.StreamId); - _target.OnDrainComplete(slot.StreamId); - slot.DisposeResources(); - _poolContext.Return(slot); - } - - private void RemoveAndReturnSlot(long streamId, BodyDrainSlot slot) - { - _activeSlots.Remove(streamId); - slot.DisposeResources(); - _poolContext.Return(slot); - } + public void Cleanup() => CancelAll(); } diff --git a/src/TurboHTTP/Protocol/Body/SerialBodyPump.cs b/src/TurboHTTP/Protocol/Body/SerialBodyPump.cs index 81dfd9060..3dc264542 100644 --- a/src/TurboHTTP/Protocol/Body/SerialBodyPump.cs +++ b/src/TurboHTTP/Protocol/Body/SerialBodyPump.cs @@ -1,145 +1,25 @@ -using Akka.Actor; +using TurboHTTP.Pooling; namespace TurboHTTP.Protocol.Body; -internal sealed class SerialBodyPump +internal sealed class SerialBodyPump : BodyPumpBase { - private readonly IBodyDrainTarget _target; - private readonly CancellationTokenSource _connectionCts; - private readonly int _chunkSize; - private readonly int _maxCapacity; - - private readonly BodyDrainSlot _slot = new(); - - private int _availableCapacity; + private readonly int _initialCredits; public SerialBodyPump( IBodyDrainTarget target, + ConnectionPoolContext poolContext, CancellationTokenSource connectionCts, - int chunkSize, - int maxCapacity) - { - _target = target; - _connectionCts = connectionCts; - _chunkSize = chunkSize; - _maxCapacity = maxCapacity; - } - - public void Register(Stream bodyStream, long? contentLength, CancellationToken requestCt) - { - var linkedCts = requestCt.CanBeCanceled - ? CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token, requestCt) - : CancellationTokenSource.CreateLinkedTokenSource(_connectionCts.Token); - _slot.Initialize(0, bodyStream, contentLength, requestCt, linkedCts); - _slot.EnsureBuffer(_chunkSize); - _availableCapacity = _maxCapacity; - TryStartRead(); - } - - public void OnCapacityAvailable() - { - if (_availableCapacity < _maxCapacity) - { - _availableCapacity++; - } - - TryStartRead(); - } - - public void ResetSyncReadCounter() - { - _slot.ResetSyncReads(); - } - - public void HandleReadComplete(int bytesRead) - { - _slot.CompleteAsyncRead(); - ProcessReadResult(bytesRead); - } - - public void HandleReadFailed(Exception reason) - { - _slot.CompleteAsyncRead(); - _target.OnDrainFailed(0, reason); - CompleteDrain(); - } - - public void HandleDrainContinue() - { - _slot.CompleteSyncRead(); - TryStartRead(); - } - - public void Cancel() + int initialCredits = 2) + : base(target, poolContext, connectionCts) { - _slot.LinkedCts?.Cancel(); - _slot.DisposeResources(); - _slot.Reset(); - _availableCapacity = 0; + _initialCredits = initialCredits; } - public void Cleanup() + public void Register(Stream bodyStream, CancellationToken requestCt) { - _slot.DisposeResources(); - _slot.Reset(); - _availableCapacity = 0; + base.Register(0, bodyStream, requestCt, _initialCredits); } - private void TryStartRead() - { - if (_availableCapacity <= 0 || _slot.IsReadInFlight || _slot.BodyStream is null) - { - return; - } - - // Starvation guard: yield after MaxSyncReadsPerDispatch consecutive sync reads - // so the actor thread can process other messages between bursts. - if (_slot.ConsecutiveSyncReads >= BodyPumpHelper.MaxSyncReadsPerDispatch) - { - _slot.ResetSyncReads(); - _slot.BeginRead(); - _target.StageActor.Tell(new DrainContinue(0), ActorRefs.NoSender); - return; - } - - _availableCapacity--; - var result = BodyPumpHelper.StartRead(_slot, _chunkSize, _target.StageActor); - - switch (result.Outcome) - { - case BodyPumpHelper.ReadOutcome.CompletedSynchronously: - ProcessReadResult(result.BytesRead); - break; - - case BodyPumpHelper.ReadOutcome.Dispatched: - // Async — HandleReadComplete/HandleReadFailed will be called. - break; - } - } - - private void ProcessReadResult(int bytesRead) - { - if (bytesRead == 0) - { - _target.EmitDataFrames(0, default, endStream: true); - CompleteDrain(); - return; - } - - _target.EmitDataFrames(0, _slot.Buffer!.Memory[..bytesRead], endStream: false); - TryStartRead(); - } - - private void CompleteDrain() - { - var wasActive = _slot.BodyStream is not null; - _slot.DisposeResources(); - _slot.Reset(); - _availableCapacity = 0; - - if (wasActive) - { - _target.OnDrainComplete(0); - } - } + public void Cleanup() => CancelAll(); } diff --git a/src/TurboHTTP/Protocol/Syntax/Http10/Client/Http10ClientStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http10/Client/Http10ClientStateMachine.cs index 49b45ebc9..6be3b7675 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http10/Client/Http10ClientStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http10/Client/Http10ClientStateMachine.cs @@ -73,7 +73,9 @@ private CancellationTokenSource EnsureConnectionCts() return _connectionCts ??= new CancellationTokenSource(); } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => false; + int IBodyDrainTarget.PreferredChunkSize => 16 * 1024; void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -86,8 +88,7 @@ void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory dat Tracing.For("Protocol").Trace(this, "HTTP/1.0 request body chunk flushed (bytes={0})", data.Length); // H1.0 has no OnOutboundFlushed — drive the pump inline. - _serialPump!.ResetSyncReadCounter(); - _serialPump.OnCapacityAvailable(); + _serialPump!.AddCredit(); } if (endStream) @@ -190,15 +191,11 @@ public void OnBodyMessage(object msg) break; case DrainReadComplete read: - _serialPump?.HandleReadComplete(read.BytesRead); + _serialPump?.HandleReadComplete(read.StreamId, read.BytesRead); break; case DrainReadFailed failed: - _serialPump?.HandleReadFailed(failed.Reason); - break; - - case DrainContinue: - _serialPump?.HandleDrainContinue(); + _serialPump?.HandleReadFailed(failed.StreamId, failed.Reason); break; } } @@ -289,8 +286,8 @@ private void EncodeRequest(HttpRequestMessage request) private void StartBodyDrain(Stream bodyStream) { - _serialPump = new SerialBodyPump(this, EnsureConnectionCts(), _options.RequestBodyChunkSize, maxCapacity: 1); - _serialPump.Register(bodyStream, contentLength: null, CancellationToken.None); + _serialPump = new SerialBodyPump(this, _poolContext, EnsureConnectionCts()); + _serialPump.Register(bodyStream, CancellationToken.None); } private void DecodeResponse(TransportBuffer buffer) diff --git a/src/TurboHTTP/Protocol/Syntax/Http10/Server/Http10ServerStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http10/Server/Http10ServerStateMachine.cs index 1b4aafa50..6cfa0afca 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http10/Server/Http10ServerStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http10/Server/Http10ServerStateMachine.cs @@ -69,7 +69,9 @@ private CancellationTokenSource EnsureConnectionCts() return _connectionCts ??= new CancellationTokenSource(); } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => false; + int IBodyDrainTarget.PreferredChunkSize => 16 * 1024; void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -83,7 +85,7 @@ void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory dat _ops.OnOutbound(TransportData.Rent(item)); Tracing.For("Protocol").Trace(this, "HTTP/1.0 response body chunk flushed (bytes={0})", data.Length); - _serialPump!.OnCapacityAvailable(); + _serialPump!.AddCredit(); } if (endStream) @@ -248,9 +250,9 @@ public void OnResponse(IFeatureCollection features) _closeAfterBody = true; } - _serialPump = new SerialBodyPump(this, EnsureConnectionCts(), 16 * 1024, maxCapacity: 1); + _serialPump = new SerialBodyPump(this, _poolContext, EnsureConnectionCts(), initialCredits: 16); EncodeDeferredResponse(ReadOnlySpan.Empty, suppressContentLength: _closeAfterBody); - _serialPump.Register(bodyStream, contentLength, CancellationToken.None); + _serialPump.Register(bodyStream, CancellationToken.None); return; } } @@ -266,8 +268,7 @@ public void OnOutboundFlushed() { if (_serialPump is not null) { - _serialPump.ResetSyncReadCounter(); - _serialPump.OnCapacityAvailable(); + _serialPump.AddCredit(); } } @@ -301,15 +302,11 @@ public void OnBodyMessage(object msg) switch (msg) { case DrainReadComplete read: - _serialPump?.HandleReadComplete(read.BytesRead); + _serialPump?.HandleReadComplete(read.StreamId, read.BytesRead); break; case DrainReadFailed failed: - _serialPump?.HandleReadFailed(failed.Reason); - break; - - case DrainContinue: - _serialPump?.HandleDrainContinue(); + _serialPump?.HandleReadFailed(failed.StreamId, failed.Reason); break; } } diff --git a/src/TurboHTTP/Protocol/Syntax/Http11/Client/Http11ClientStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http11/Client/Http11ClientStateMachine.cs index 47c950c37..28a45a94c 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http11/Client/Http11ClientStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http11/Client/Http11ClientStateMachine.cs @@ -87,7 +87,9 @@ private CancellationTokenSource EnsureConnectionCts() return _connectionCts ??= new CancellationTokenSource(); } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => _ops.HasPendingDemand; + int IBodyDrainTarget.PreferredChunkSize => _options.RequestBodyChunkSize; void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -298,15 +300,11 @@ public void OnBodyMessage(object msg) break; case DrainReadComplete read: - _serialPump?.HandleReadComplete(read.BytesRead); + _serialPump?.HandleReadComplete(read.StreamId, read.BytesRead); break; case DrainReadFailed failed: - _serialPump?.HandleReadFailed(failed.Reason); - break; - - case DrainContinue: - _serialPump?.HandleDrainContinue(); + _serialPump?.HandleReadFailed(failed.StreamId, failed.Reason); break; } } @@ -315,8 +313,7 @@ public void OnOutboundFlushed() { if (_serialPump is not null) { - _serialPump.ResetSyncReadCounter(); - _serialPump.OnCapacityAvailable(); + _serialPump.AddCredit(); } } @@ -491,8 +488,8 @@ private void StartBodyDrain(Stream bodyStream, long? contentLength, Version http _isChunked = contentLength is null && !httpVersion.Equals(HttpVersion.Version10); Tracing.For("Protocol").Debug(this, "StartBodyDrain: chunked={0}, contentLength={1}", _isChunked, contentLength); - _serialPump = new SerialBodyPump(this, EnsureConnectionCts(), _options.RequestBodyChunkSize, maxCapacity: 2); - _serialPump.Register(bodyStream, contentLength, CancellationToken.None); + _serialPump = new SerialBodyPump(this, _poolContext, EnsureConnectionCts()); + _serialPump.Register(bodyStream, CancellationToken.None); } private void HandleDisconnect(TransportDisconnected disconnect) diff --git a/src/TurboHTTP/Protocol/Syntax/Http11/Server/Http11ServerStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http11/Server/Http11ServerStateMachine.cs index ee532472c..fed874bc4 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http11/Server/Http11ServerStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http11/Server/Http11ServerStateMachine.cs @@ -109,7 +109,9 @@ private CancellationTokenSource EnsureConnectionCts() return _connectionCts ??= new CancellationTokenSource(); } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => _ops.HasPendingDemand; + int IBodyDrainTarget.PreferredChunkSize => _bodyEncoderOptions.ChunkSize; void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -140,7 +142,7 @@ void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory dat if (!endStream && _serialPump is not null) { - _serialPump.OnCapacityAvailable(); + _serialPump.AddCredit(); } if (endStream) @@ -488,8 +490,8 @@ public void OnResponse(IFeatureCollection features) var bodyStream = turboBody.GetResponseStream(); _serialPump = - new SerialBodyPump(this, EnsureConnectionCts(), _bodyEncoderOptions.ChunkSize, maxCapacity: 2); - _serialPump.Register(bodyStream, contentLength, CancellationToken.None); + new SerialBodyPump(this, _poolContext, EnsureConnectionCts(), initialCredits: 16); + _serialPump.Register(bodyStream, CancellationToken.None); } else { @@ -675,15 +677,11 @@ public void OnBodyMessage(object msg) switch (msg) { case DrainReadComplete read: - _serialPump?.HandleReadComplete(read.BytesRead); + _serialPump?.HandleReadComplete(read.StreamId, read.BytesRead); break; case DrainReadFailed failed: - _serialPump?.HandleReadFailed(failed.Reason); - break; - - case DrainContinue: - _serialPump?.HandleDrainContinue(); + _serialPump?.HandleReadFailed(failed.StreamId, failed.Reason); break; } } @@ -692,8 +690,7 @@ public void OnOutboundFlushed() { if (_serialPump is not null) { - _serialPump.ResetSyncReadCounter(); - _serialPump.OnCapacityAvailable(); + _serialPump.AddCredit(); } } diff --git a/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientSessionManager.cs b/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientSessionManager.cs index e5d8988c2..273394187 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientSessionManager.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientSessionManager.cs @@ -81,8 +81,7 @@ public Http2ClientSessionManager( _decoderOptions.InitialStreamWindowSize, scaler, clock); - var chunkSize = Math.Min(options.RequestBodyChunkSize, 16 * 1024); - _pump = new FlowControlledBodyPump(this, _flow, _poolContext, _connectionCts, chunkSize, hardCap: 16); + _pump = new FlowControlledBodyPump(this, _flow, _poolContext, _connectionCts); // Outgoing frame size starts at the RFC 9113 default (16,384) and is raised only when the // server advertises a larger SETTINGS_MAX_FRAME_SIZE. The client's own MaxFrameSize option // is a receive-side advertisement (sent in the preface), not a send-side limit. @@ -242,9 +241,17 @@ public void EncodeRequest(HttpRequestMessage request) return; } + if (contentLength == 0) + { + // Empty body: emit END_STREAM directly without involving the pump (spec invariant 7). + ((IBodyDrainTarget)this).EmitDataFrames(streamId, default, endStream: true); + CloseStream(streamId); + return; + } + state.MarkBodyDrainActive(); _drainContentOwners[streamId] = request.Content!; - _pump!.Register(streamId, bodyStream!, contentLength, request.GetCancellationToken()); + _pump!.Register(streamId, bodyStream!, request.GetCancellationToken(), initialCredits: 16); } private void EmitBodyDirect(int streamId, StreamState state, Memory body) @@ -282,7 +289,8 @@ private void EmitBodyDirect(int streamId, StreamState state, Memory body) // Window exhausted before all data sent: hand the remainder to the pump // which will emit it when the send window opens up via WINDOW_UPDATE. state.MarkBodyDrainActive(); - _pump!.RegisterWithLimbo(streamId, body[sent..], CancellationToken.None); + var remainderBytes = body[sent..].ToArray(); + _pump!.Register(streamId, new MemoryStream(remainderBytes, writable: false), CancellationToken.None, initialCredits: 16); } private bool TrySerializeBodyDirect(HttpContent content, int streamId, StreamState state, int bodyLength) @@ -488,7 +496,14 @@ public void Cleanup() ReleaseAllStreamState(); } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + public void OnOutboundFlushed() + { + _pump?.AddCredit(); + } + + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => _ops.HasPendingDemand; + int IBodyDrainTarget.PreferredChunkSize => _requestEncoder.MaxFrameSize; void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -506,6 +521,11 @@ void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory dat { EmitFrame(new DataFrame(streamId, remaining, endStream)); } + + if (!endStream && !data.IsEmpty) + { + _pump?.AddCredit(); + } } void IBodyDrainTarget.OnDrainComplete(int streamId) @@ -872,10 +892,6 @@ public void OnBodyMessage(object msg) case DrainReadFailed failed: _pump?.HandleReadFailed(failed.StreamId, failed.Reason); break; - - case DrainContinue cont: - _pump?.HandleDrainContinue(cont.StreamId); - break; } } diff --git a/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientStateMachine.cs index fd81715e5..ba839c4b4 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http2/Client/Http2ClientStateMachine.cs @@ -178,6 +178,11 @@ public void OnRequestCancelled(HttpRequestMessage request) } } + public void OnOutboundFlushed() + { + _clientSession.OnOutboundFlushed(); + } + public void OnBodyMessage(object msg) => _clientSession.OnBodyMessage(msg); public void Cleanup() => _clientSession.Cleanup(); diff --git a/src/TurboHTTP/Protocol/Syntax/Http2/FlowController.cs b/src/TurboHTTP/Protocol/Syntax/Http2/FlowController.cs index 128da2977..2c21d571d 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http2/FlowController.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http2/FlowController.cs @@ -213,6 +213,26 @@ public void RemoveStreamSendWindow(int streamId) _streamSendWindows.Remove(streamId); } + public void Reserve(int streamId, int amount) + { + _connectionSendWindow -= amount; + if (_streamSendWindows.TryGetValue(streamId, out var current)) + { + _streamSendWindows[streamId] = current - amount; + } + } + + public void Refund(int streamId, int amount) + { + if (amount <= 0) return; + + _connectionSendWindow += amount; + if (_streamSendWindows.TryGetValue(streamId, out var current)) + { + _streamSendWindows[streamId] = current + amount; + } + } + public void ApplyInitialWindowSizeDelta(long delta) { _initialSendStreamWindow += delta; diff --git a/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerSessionManager.cs b/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerSessionManager.cs index 916290580..50a0180fa 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerSessionManager.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerSessionManager.cs @@ -103,8 +103,7 @@ public Http2ServerSessionManager( scaler, _clock); _tracker = new StreamTracker(initialNextStreamId: 1, options.MaxConcurrentStreams); - var chunkSize = options.ToBodyEncoderOptions().ChunkSize; - _pump = new FlowControlledBodyPump(this, _flow, _poolContext, _connectionCts, chunkSize, hardCap: 16); + _pump = new FlowControlledBodyPump(this, _flow, _poolContext, _connectionCts); _maxRequestBodySize = options.Limits.MaxRequestBodySize; _maxResetStreamsPerWindow = options.Limits.MaxResetStreamsPerWindow; _bodyEncoderOptions = options.ToBodyEncoderOptions(); @@ -369,7 +368,7 @@ public void OnResponse(IFeatureCollection features) state.MarkBodyDrainActive(); _pump!.Register(streamId, new MemoryStream(segment.Array!, segment.Offset, segment.Count, writable: false), - bufferedBody.Length, CancellationToken.None); + CancellationToken.None, initialCredits: 16); return; } @@ -386,7 +385,7 @@ public void OnResponse(IFeatureCollection features) var bodyStream = turboBody.GetResponseStream(); state.MarkBodyDrainActive(); - _pump!.Register(streamId, bodyStream, contentLength, CancellationToken.None); + _pump!.Register(streamId, bodyStream, CancellationToken.None, initialCredits: 16); Tracing.For("Protocol").Debug(this, "HTTP/2: response body drain started (stream={0})", streamId); } @@ -421,10 +420,6 @@ public void OnBodyMessage(object msg) _pump?.HandleReadFailed(failed.StreamId, failed.Reason); break; - case DrainContinue cont: - _pump?.HandleDrainContinue(cont.StreamId); - break; - case StreamBodyConsumed consumed: if (_deferredStreamIncrements.TryGetValue(consumed.StreamId, out var inc) && inc > 0) { @@ -939,14 +934,22 @@ private void SendBufferedBodyWithFlowControl(int streamId, StreamState state, Re // Hand the remainder to the scheduler which will emit it when WINDOW_UPDATE arrives. state.MarkBodyDrainActive(); - _pump!.RegisterWithLimbo(streamId, remainder, CancellationToken.None); + var remainderBytes = remainder.ToArray(); + _pump!.Register(streamId, new MemoryStream(remainderBytes, writable: false), CancellationToken.None, initialCredits: 16); Tracing.For("Protocol").Debug(this, "HTTP/2: buffered body flow-controlled (stream={0}, sent={1}, queued={2})", streamId, sent, body.Length - sent); } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + public void OnOutboundFlushed() + { + _pump?.AddCredit(); + } + + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => _ops.HasPendingDemand; + int IBodyDrainTarget.PreferredChunkSize => _responseEncoder.MaxFrameSize; void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory data, bool endStream) { @@ -956,6 +959,11 @@ void IBodyDrainTarget.EmitDataFrames(int streamId, ReadOnlyMemory dat } EmitBufferedDataFrames(streamId, data, endStream: false); + + if (!endStream) + { + _pump?.AddCredit(); + } } void IBodyDrainTarget.OnDrainComplete(int streamId) diff --git a/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerStateMachine.cs index 39399b5b8..b438eb78c 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http2/Server/Http2ServerStateMachine.cs @@ -144,6 +144,11 @@ public void OnTimerFired(string name) } } + public void OnOutboundFlushed() + { + _sessionManager.OnOutboundFlushed(); + } + public void OnBodyMessage(object msg) => _sessionManager.OnBodyMessage(msg); private void ScheduleKeepAlivePing() diff --git a/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientSessionManager.cs b/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientSessionManager.cs index b5c778a58..5a5b52eb2 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientSessionManager.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientSessionManager.cs @@ -150,11 +150,18 @@ public void EncodeRequest(HttpRequestMessage request) return; } + if (contentLength == 0) + { + // Empty body: emit END_STREAM directly without involving the pump (spec invariant 7). + EmitBufferedDataFrames(streamId, default, endStream: true); + return; + } + var state = _streamManager.GetOrCreateStreamState(streamId); state.MarkBodyDrainActive(); _drainContentOwners[streamId] = request.Content!; - _pump ??= new MultiplexedBodyPump(this, _poolContext, _connectionCts, _options.RequestBodyChunkSize); - _pump.Register(streamId, bodyStream!, contentLength, CancellationToken.None); + _pump ??= new MultiplexedBodyPump(this, _poolContext, _connectionCts); + _pump.Register(streamId, bodyStream!, CancellationToken.None, initialCredits: 16); } public void OnBodyMessage(object msg) @@ -168,10 +175,6 @@ public void OnBodyMessage(object msg) case DrainReadFailed failed: _pump?.HandleReadFailed(failed.StreamId, failed.Reason); break; - - case DrainContinue cont: - _pump?.HandleDrainContinue(cont.StreamId); - break; } } @@ -368,11 +371,23 @@ private bool TrySerializeBodyDirect(HttpContent content, long streamId, int body return true; } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + public void OnOutboundFlushed() + { + _pump?.AddCredit(); + } + + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => _ops.HasPendingDemand; + int IBodyDrainTarget.PreferredChunkSize => 16 * 1024; void IBodyDrainTarget.EmitDataFrames(long streamId, ReadOnlyMemory data, bool endStream) { EmitBufferedDataFrames(streamId, data, endStream); + + if (!endStream && !data.IsEmpty) + { + _pump?.AddCredit(); + } } private void EmitBufferedDataFrames(long streamId, ReadOnlyMemory body, bool endStream) diff --git a/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientStateMachine.cs index 803324c69..3782662c1 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http3/Client/Http3ClientStateMachine.cs @@ -236,6 +236,11 @@ public void OnRequestCancelled(HttpRequestMessage request) } } + public void OnOutboundFlushed() + { + _clientSession.OnOutboundFlushed(); + } + public void OnBodyMessage(object msg) { _clientSession.OnBodyMessage(msg); diff --git a/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerSessionManager.cs b/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerSessionManager.cs index 8028afbb0..6d79a11f7 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerSessionManager.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerSessionManager.cs @@ -233,8 +233,8 @@ public void OnResponse(IFeatureCollection features) var bodyStream = turboBody.GetResponseStream(); state.MarkBodyDrainActive(); - _pump ??= new MultiplexedBodyPump(this, _poolContext, _connectionCts, _responseBodyChunkSize); - _pump.Register(streamId, bodyStream, contentLength, CancellationToken.None); + _pump ??= new MultiplexedBodyPump(this, _poolContext, _connectionCts); + _pump.Register(streamId, bodyStream, CancellationToken.None, initialCredits: 16); Tracing.For("Protocol").Debug(this, "HTTP/3: response body drain started (stream={0})", streamId); } @@ -268,10 +268,6 @@ public void OnBodyMessage(object msg) case DrainReadFailed failed: _pump?.HandleReadFailed(failed.StreamId, failed.Reason); break; - - case DrainContinue cont: - _pump?.HandleDrainContinue(cont.StreamId); - break; } } @@ -761,13 +757,25 @@ private void ReturnDecoder(FrameDecoder decoder) } } - IActorRef IBodyDrainTarget.StageActor => _ops.StageActor; + public void OnOutboundFlushed() + { + _pump?.AddCredit(); + } + + IActorRef IBodyDrainTarget.PipeToTarget => _ops.StageActor; + bool IBodyDrainTarget.HasPendingDemand => _ops.HasPendingDemand; + int IBodyDrainTarget.PreferredChunkSize => 16 * 1024; void IBodyDrainTarget.EmitDataFrames(long streamId, ReadOnlyMemory data, bool endStream) { if (!data.IsEmpty) { EmitBufferedDataFrames(streamId, data, endStream); + + if (!endStream) + { + _pump?.AddCredit(); + } } } diff --git a/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerStateMachine.cs b/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerStateMachine.cs index 66d07cb8a..8ace5c0a5 100644 --- a/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerStateMachine.cs +++ b/src/TurboHTTP/Protocol/Syntax/Http3/Server/Http3ServerStateMachine.cs @@ -104,6 +104,11 @@ public void OnTimerFired(string name) } } + public void OnOutboundFlushed() + { + _sessionManager.OnOutboundFlushed(); + } + public void OnBodyMessage(object msg) { _sessionManager.OnBodyMessage(msg); diff --git a/src/TurboHTTP/Streams/Stages/Client/HttpConnectionStageLogic.cs b/src/TurboHTTP/Streams/Stages/Client/HttpConnectionStageLogic.cs index fb644068b..4fb20742e 100644 --- a/src/TurboHTTP/Streams/Stages/Client/HttpConnectionStageLogic.cs +++ b/src/TurboHTTP/Streams/Stages/Client/HttpConnectionStageLogic.cs @@ -237,6 +237,8 @@ void IClientStageOperations.OnOutbound(ITransportOutbound item) IActorRef IClientStageOperations.StageActor => _stageActor; + bool IClientStageOperations.HasPendingDemand => _outboundQueue.Count == 0 && IsAvailable(_outNetwork); + private void OnRequestCancelled(HttpRequestMessage request) { if (_ctRegistrations.Remove(request, out var reg)) diff --git a/src/TurboHTTP/Streams/Stages/Client/IClientStageOperations.cs b/src/TurboHTTP/Streams/Stages/Client/IClientStageOperations.cs index 7a7decff0..147c98051 100644 --- a/src/TurboHTTP/Streams/Stages/Client/IClientStageOperations.cs +++ b/src/TurboHTTP/Streams/Stages/Client/IClientStageOperations.cs @@ -10,4 +10,5 @@ internal interface IClientStageOperations void OnScheduleTimer(string name, TimeSpan duration); void OnCancelTimer(string name); IActorRef StageActor { get; } + bool HasPendingDemand => false; } \ No newline at end of file diff --git a/src/TurboHTTP/Streams/Stages/Server/HttpConnectionServerStageLogic.cs b/src/TurboHTTP/Streams/Stages/Server/HttpConnectionServerStageLogic.cs index 184a1e01e..ea95394e3 100644 --- a/src/TurboHTTP/Streams/Stages/Server/HttpConnectionServerStageLogic.cs +++ b/src/TurboHTTP/Streams/Stages/Server/HttpConnectionServerStageLogic.cs @@ -468,6 +468,8 @@ void IServerStageOperations.OnCancelTimer(string name) IActorRef IServerStageOperations.StageActor => _stageActor; + bool IServerStageOperations.HasPendingDemand => _outboundQueue.Count == 0 && IsAvailable(_outNetwork); + IMaterializer IServerStageOperations.Materializer => Materializer; IServiceProvider? IServerStageOperations.Services => _services; diff --git a/src/TurboHTTP/Streams/Stages/Server/IServerStageOperations.cs b/src/TurboHTTP/Streams/Stages/Server/IServerStageOperations.cs index 93b22dbb5..259ae7185 100644 --- a/src/TurboHTTP/Streams/Stages/Server/IServerStageOperations.cs +++ b/src/TurboHTTP/Streams/Stages/Server/IServerStageOperations.cs @@ -22,4 +22,5 @@ internal interface IServerStageOperations TlsHandshakeFeature? TlsHandshakeFeature => null; ConnectionPoolContext? PoolContext => null; void OnResponseBodyComplete(IFeatureCollection features) { } + bool HasPendingDemand => false; } \ No newline at end of file