fix: eliminate shared-pool poisoning behind H2 desyncs + trap-wire remaining flakes

src/GaudiHTTP.IntegrationTests.Client/H2/LargeDownloadRegressionSpec.cs

@@ -2,6 +2,7 @@
 using GaudiHTTP.Client;
 using GaudiHTTP.IntegrationTests.Client.Shared;
 using GaudiHTTP.Tests.Shared;
+using Servus.Diagnostics;
 
 namespace GaudiHTTP.IntegrationTests.Client.H2;
 
@@ -10,7 +11,9 @@ namespace GaudiHTTP.IntegrationTests.Client.H2;
 /// (GaudiClientDownloadBenchmarks [ConcurrencyLevel=1, DownloadBytes=8388608, HttpVersion=2.0] → NA,
 /// "System.TimeoutException: The operation has timed out"). A SINGLE 8 MB response over one H2 stream
 /// hung to the 120 s WaitAsync, while 1 MB over H2 — and 8 MB over H1.1 and H3 — all completed.
-/// Suspected receive-path flow-control / WINDOW_UPDATE stall on a single large stream.
+/// Root cause (2026-07): shared-pool buffer corruption — a double dispose returned another owner's
+/// array to the process-wide pool, desyncing the H2 frame decoder mid-download. Flow control /
+/// WINDOW_UPDATE was ruled out.
 ///
 /// 1 MB is included first as a sanity check (it completes in the benchmark); the 8 MB download is the
 /// configuration that hung.
@@ -27,9 +30,17 @@ public LargeDownloadRegressionSpec(ServerContainerFixture server, ActorSystemFix
     // matching the benchmark's ConcurrencyLevel=1 over the default pool.
     protected override ProtocolVariant? Variant => null;
 
+    // Stall post-mortem: capture per-frame Protocol traces (DATA in/out, WINDOW_UPDATE in/out,
+    // pause/resume) in a ring buffer; DownloadAsync dumps it to a file when the 30 s budget trips.
+    private static readonly RingBufferTraceListener TraceRing =
+        new(capacity: 128 * 1024, category => category is "Protocol" or "Stage" or "Transport");
+
     [Fact(Timeout = 180_000)]
     public async Task LargeDownload_should_complete_8MB_body_over_single_H2_stream()
     {
+        // Additive registration: Tracing.Configure would REPLACE the logger bridge installed by
+        // ActorSystemFixture and swallow every other test's warnings while this spec runs.
+        using var traceScope = TestTracing.Root.Add(TraceRing);
         await using var helper = CreateClient(
             new ProtocolVariant(TestHttpVersion.H2, tls: true),
             configureOptions: o => o.Http2.MaxConnectionsPerServer = 1);
@@ -75,9 +86,28 @@ private async Task DownloadAsync(IGaudiHttpClient client, int size)
         }
         catch (OperationCanceledException) when (cts.IsCancellationRequested && !CancellationToken.IsCancellationRequested)
         {
+            var dumpPath = Path.Combine(Path.GetTempPath(),
+                $"h2-large-download-stall-{DateTime.UtcNow:yyyyMMdd_HHmmss}.trace.log");
+            await File.WriteAllTextAsync(dumpPath, TraceRing.Dump(), CancellationToken.None);
+
             Assert.Fail(
-                $"REPRO: a {size / (1024 * 1024)} MB HTTP/2 download did not complete within 30 s — " +
-                "the receive path stalls on a large single stream (suspected missing/stuck WINDOW_UPDATE).");
+                $"REPRO: a {size / (1024 * 1024)} MB HTTP/2 download did not complete within 30 s. " +
+                "Historically caused by shared-pool buffer corruption desyncing the frame decoder " +
+                "(flow control / WINDOW_UPDATE was ruled out) — analyze the dump, don't guess. " +
+                $"Protocol trace ring dumped to: {dumpPath}");
+        }
+        catch (Exception ex) when (ex is not Xunit.Sdk.SkipException)
+        {
+            // Receive-path failures now fail fast (the body reader faults on connection loss instead
+            // of stalling to the 30 s budget above), so the post-mortem ring must also be dumped for
+            // this path — otherwise the fast failure hides exactly the frame trace the stall dump
+            // was added to capture. The original exception propagates unchanged.
+            var dumpPath = Path.Combine(Path.GetTempPath(),
+                $"h2-large-download-fault-{DateTime.UtcNow:yyyyMMdd_HHmmss}.trace.log");
+            await File.WriteAllTextAsync(dumpPath, TraceRing.Dump(), CancellationToken.None);
+            await Console.Error.WriteLineAsync(
+                $"[LargeDownloadRegressionSpec] {ex.GetType().Name}: {ex.Message} — protocol trace ring dumped to: {dumpPath}");
+            throw;
         }
     }
 }

src/GaudiHTTP.IntegrationTests.Client/H2/SendAsyncHighConcurrencySpec.cs

@@ -23,12 +23,14 @@ namespace GaudiHTTP.IntegrationTests.Client.H2;
 /// over H2 against Kestrel. Uses graduated concurrency levels with Senf tracing to
 /// pinpoint where the pipeline stalls.
 /// </summary>
+[Collection("HighConcurrency")]
 public sealed class SendAsyncHighConcurrencySpec : IAsyncLifetime
 {
     private WebApplication? _kestrelApp;
     private IGaudiHttpClient? _client;
     private Microsoft.Extensions.DependencyInjection.ServiceProvider? _clientProvider;
     private string _baseUri = string.Empty;
+    private IDisposable? _traceScope;
 
     private CancellationToken CT => TestContext.Current.CancellationToken;
 
@@ -77,7 +79,7 @@ public async ValueTask InitializeAsync()
         _baseUri = $"http://127.0.0.1:{port}";
 
         // --- GaudiHTTP client (matches benchmark ClientHelper config) ---
-        ConfigureTracing();
+        _traceScope = ConfigureTracing();
 
         var services = new ServiceCollection();
 
@@ -138,7 +140,9 @@ public async ValueTask DisposeAsync()
             await _clientProvider.DisposeAsync();
         }
 
-        Tracing.Disable();
+        // Remove only our own listener — Tracing.Disable() would kill tracing for every other
+        // collection still running in this process.
+        _traceScope?.Dispose();
     }
 
     /// <summary>
@@ -734,7 +738,7 @@ private sealed class Counters
         public int Failed;
     }
 
-    private static void ConfigureTracing()
+    private static IDisposable ConfigureTracing()
     {
         ThreadPool.GetMinThreads(out var w, out var io);
         ThreadPool.SetMinThreads(Math.Max(w, 1024), Math.Max(io, 1024));
@@ -745,7 +749,11 @@ private static void ConfigureTracing()
             b.SetMinimumLevel(LogLevel.Debug);
         });
 
-        Tracing.Configure(
+        // Additive registration: Tracing.Configure would REPLACE the logger bridge installed by
+        // ActorSystemFixture for the other collections in this process. Configure(Root) is
+        // idempotent (same singleton), so this works whether or not a fixture ran first.
+        Tracing.Configure(TestTracing.Root, TraceLevel.Trace);
+        return TestTracing.Root.Add(
             new Diagnostics.LoggerTraceListener(loggerFactory),
             TraceLevel.Warning);
     }

src/GaudiHTTP.IntegrationTests.Client/Shared/Collections.cs

@@ -49,4 +49,10 @@ public sealed class StreamingIntegrationCollection;
 public sealed class TimingIntegrationCollection;
 
 [CollectionDefinition("Cancellation")]
-public sealed class CancellationIntegrationCollection;
+public sealed class CancellationIntegrationCollection;
+
+// Runs exclusively: the spec boots its own in-process Kestrel per test and drives 512
+// concurrent requests. Sharing the process with parallel collections causes sporadic
+// multi-second stalls (random victims on both sides).
+[CollectionDefinition("HighConcurrency", DisableParallelization = true)]
+public sealed class HighConcurrencyCollection;

src/GaudiHTTP.IntegrationTests.Client/Shared/HttpbinEndpoints.cs

@@ -118,7 +118,9 @@ private static async Task HandleSetCookies(HttpContext ctx)
         {
             var sanitizedKey = SanitizeCookieToken(kvp.Key);
             var sanitizedValue = SanitizeCookieToken(kvp.Value.ToString());
-            ctx.Response.Cookies.Append(sanitizedKey, sanitizedValue, new CookieOptions { Path = "/", Secure = true, HttpOnly = true, SameSite = SameSiteMode.Lax });
+            // Secure only on TLS: httpbin sets no Secure attribute; an unconditional Secure=true
+            // makes the plaintext H1 variants unpassable (the client refuses Secure cookies over http).
+            ctx.Response.Cookies.Append(sanitizedKey, sanitizedValue, new CookieOptions { Path = "/", Secure = ctx.Request.IsHttps, HttpOnly = true, SameSite = SameSiteMode.Lax });
         }
         ctx.Response.StatusCode = 302;
         ctx.Response.Redirect("/cookies", permanent: false);
@@ -474,7 +476,12 @@ private static async Task HandleAbsoluteRedirect(HttpContext ctx, int n)
             }
             else if (ctx.Connection.LocalIpAddress != null)
             {
-                hostIp = ctx.Connection.LocalIpAddress.ToString();
+                // Loopback → "localhost": matches the TLS cert and avoids emitting an
+                // unbracketed IPv6 authority ("::1:port") now that the backend also
+                // listens on ::1.
+                hostIp = System.Net.IPAddress.IsLoopback(ctx.Connection.LocalIpAddress)
+                    ? "localhost"
+                    : ctx.Connection.LocalIpAddress.ToString();
             }
 
             var port = ctx.Connection.LocalPort;

src/GaudiHTTP.IntegrationTests.Client/Shared/IntegrationSpecBase.cs

@@ -40,6 +40,10 @@ public ValueTask InitializeAsync()
 
     public async ValueTask DisposeAsync()
     {
+        // Dump before tearing down the client so the ring still shows the stalled request's frames
+        // rather than teardown noise.
+        await FaultTraceDump.DumpIfTestFailedAsync();
+
         if (_helper is not null)
         {
             await _helper.DisposeAsync();

src/GaudiHTTP.IntegrationTests.Client/Shared/KestrelTestBackend.cs

@@ -1,5 +1,6 @@
 using System.Net;
 using System.Net.Quic;
+using System.Net.Sockets;
 using System.Security.Cryptography.X509Certificates;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
@@ -27,29 +28,57 @@ public async Task StartAsync()
         var cert = LoadCertificate();
         var quicSupported = QuicListener.IsSupported;
 
-        var builder = WebApplication.CreateBuilder();
-        builder.Logging.ClearProviders();
-
-        builder.WebHost.ConfigureKestrel(kestrel =>
+        // The reserve→listen gap is a race, and lingering TIME_WAIT pairs from a previous
+        // test-host process can still fail Kestrel's explicit bind: retry with FRESH ports
+        // instead of failing the whole suite (an AddressInUse here wipes out every test).
+        for (var attempt = 0; ; attempt++)
         {
-            kestrel.Listen(IPAddress.Loopback, 0, listenOptions =>
-            {
-                listenOptions.Protocols = HttpProtocols.Http1;
-            });
-
-            kestrel.Listen(IPAddress.Loopback, 0, listenOptions =>
+            // Listen on BOTH loopback families for each port: TLS clients resolve "localhost",
+            // try ::1 first, and Windows silently drops SYNs to non-listening ::1 ports (~2s
+            // penalty per connect, >12s under load) instead of refusing them. IPv4-only
+            // listeners therefore cause sporadic test timeouts. Ports are pre-allocated
+            // because Kestrel cannot share a dynamic port across two Listen calls.
+            var httpPort = ReserveLoopbackPort();
+            var httpsPort = ReserveLoopbackPort();
+
+            var builder = WebApplication.CreateBuilder();
+            builder.Logging.ClearProviders();
+
+            builder.WebHost.ConfigureKestrel(kestrel =>
             {
-                listenOptions.Protocols = quicSupported
-                    ? HttpProtocols.Http1AndHttp2AndHttp3
-                    : HttpProtocols.Http1AndHttp2;
-                listenOptions.UseHttps(cert);
+                foreach (var address in new[] { IPAddress.Loopback, IPAddress.IPv6Loopback })
+                {
+                    kestrel.Listen(address, httpPort, listenOptions =>
+                    {
+                        listenOptions.Protocols = HttpProtocols.Http1;
+                    });
+
+                    kestrel.Listen(address, httpsPort, listenOptions =>
+                    {
+                        listenOptions.Protocols = quicSupported
+                            ? HttpProtocols.Http1AndHttp2AndHttp3
+                            : HttpProtocols.Http1AndHttp2;
+                        listenOptions.UseHttps(cert);
+                    });
+                }
             });
-        });
 
-        _app = builder.Build();
-        HttpbinEndpoints.Map(_app);
+            var app = builder.Build();
+            HttpbinEndpoints.Map(app);
 
-        await _app.StartAsync();
+            try
+            {
+                await app.StartAsync();
+                _app = app;
+                break;
+            }
+            catch (IOException ex) when (attempt < 4)
+            {
+                await Console.Error.WriteLineAsync(
+                    $"[KestrelTestBackend] bind failed on ports {httpPort}/{httpsPort} (attempt {attempt + 1}): {ex.Message} — retrying with fresh ports");
+                await app.DisposeAsync();
+            }
+        }
 
         ResolvePortsFromServer(_app);
 
@@ -76,6 +105,35 @@ public async ValueTask DisposeAsync()
         }
     }
 
+    private static int ReserveLoopbackPort()
+    {
+        // Do NOT bind port 0 to pick a port: Windows' ephemeral allocator hands back the most
+        // recently FREED port, so consecutive test-host runs converge on the same port whose
+        // TIME_WAIT connection pairs from the previous run then fail Kestrel's explicit bind
+        // for minutes (observed as full-suite AddressInUse wipeouts). Pick a random port below
+        // the dynamic client range (49152+) instead, so client-connection churn never collides,
+        // and verify it explicitly binds on BOTH loopback families — the same bind Kestrel does.
+        for (var attempt = 0; attempt < 50; attempt++)
+        {
+            var port = Random.Shared.Next(20000, 45000);
+
+            try
+            {
+                using var probe = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
+                probe.Bind(new IPEndPoint(IPAddress.Loopback, port));
+                using var probe6 = new Socket(AddressFamily.InterNetworkV6, SocketType.Stream, ProtocolType.Tcp);
+                probe6.Bind(new IPEndPoint(IPAddress.IPv6Loopback, port));
+                return port;
+            }
+            catch (SocketException)
+            {
+                // port taken (or excluded range) on either family — try another
+            }
+        }
+
+        throw new InvalidOperationException("Could not reserve a loopback port free on both address families.");
+    }
+
     private void ResolvePortsFromServer(WebApplication app)
     {
         var server = app.Services.GetRequiredService<IServer>();

src/GaudiHTTP.IntegrationTests.Client/Shared/ServerContainerFixture.cs

@@ -14,7 +14,7 @@ public sealed class ServerContainerFixture : Xunit.IAsyncLifetime
 
     public async ValueTask InitializeAsync()
     {
-        var mode = Environment.GetEnvironmentVariable("GaudiHttp_TEST_BACKEND")?.ToLowerInvariant();
+        var mode = Environment.GetEnvironmentVariable("GAUDIHTTP_TEST_BACKEND")?.ToLowerInvariant();
 
         _backend = mode switch
         {
@@ -42,7 +42,7 @@ public async ValueTask DisposeAsync()
             if (required)
             {
                 throw new InvalidOperationException(
-                    "GaudiHttp_TEST_BACKEND=docker but Docker is not available.");
+                    "GAUDIHTTP_TEST_BACKEND=docker but Docker is not available.");
             }
 
             return null;

src/GaudiHTTP.IntegrationTests.End2End/H11/ServerStartupReliabilitySpec.cs

@@ -10,6 +10,7 @@
 using Servus.Akka.Transport;
 using Servus.Diagnostics;
 using GaudiHTTP.Server;
+using GaudiHTTP.Tests.Shared;
 
 namespace GaudiHTTP.IntegrationTests.End2End.H11;
 
@@ -19,7 +20,10 @@ public sealed class ServerStartupReliabilitySpec
     public async Task Server_with_three_listeners_should_respond_to_first_h11_request()
     {
         ThreadPool.SetMinThreads(1024, 1024);
-        Servus.Senf.Tracing.Configure(new StderrTraceListener(), TraceLevel.Debug);
+        // Additive registration via the shared composite root — a direct Tracing.Configure would
+        // replace any listener another spec in this process installed (and vice versa).
+        Servus.Senf.Tracing.Configure(TestTracing.Root, TraceLevel.Trace);
+        using var traceScope = TestTracing.Root.Add(new StderrTraceListener(), TraceLevel.Debug);
 
         var cert = GenerateSelfSignedCert();
         try

src/GaudiHTTP.IntegrationTests.End2End/Shared/End2EndSpecBase.cs

@@ -15,6 +15,7 @@
 using Servus.Akka.Transport;
 using GaudiHTTP.Client;
 using GaudiHTTP.Server;
+using GaudiHTTP.Tests.Shared;
 using QuicListenerOptionsServus = Servus.Akka.Transport.QuicListenerOptions;
 
 namespace GaudiHTTP.IntegrationTests.End2End.Shared;
@@ -152,6 +153,9 @@ public async ValueTask InitializeAsync()
 
     public virtual async ValueTask DisposeAsync()
     {
+        // Dump before teardown so the ring still shows the failed request's frames.
+        await FaultTraceDump.DumpIfTestFailedAsync();
+
         _client?.Dispose();
 
         if (_app is not null)

src/GaudiHTTP.IntegrationTests.Server/Shared/ServerSpecBase.cs

@@ -6,6 +6,7 @@
 using Microsoft.AspNetCore.Hosting.Server.Features;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using GaudiHTTP.Tests.Shared;
 
 namespace GaudiHTTP.IntegrationTests.Server.Shared;
 
@@ -44,6 +45,9 @@ public virtual async ValueTask InitializeAsync()
 
     public virtual async ValueTask DisposeAsync()
     {
+        // Dump before teardown so the ring still shows the failed request's frames.
+        await FaultTraceDump.DumpIfTestFailedAsync();
+
         _client?.Dispose();
         if (_app is not null)
         {