Add exception handler on HTTP/2 parent channel to suppress WARN logs

In HTTP/2, reactor-netty multiplexes streams on a shared parent TCP connection.
The parent channel pipeline has no ChannelOperationsHandler (unlike HTTP/1.1),
so TCP-level exceptions like Connection reset by peer (ECONNRESET) propagate to
Netty's TailContext, which logs them as WARN.

This adds Http2ParentChannelExceptionHandler to the parent channel via
doOnConnected (accessing channel.parent()). The handler consumes exceptions
at DEBUG level WITHOUT closing the channel or altering connection lifecycle,
matching HTTP/1.1 logging behavior.

Changes:
- Handler logs cause.toString() (not getMessage()) for null-safe diagnostics
- Defensive try-catch for duplicate handler name on concurrent stream creation
- Before/after verified with EmbeddedChannel unit tests

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: jeet1995

sdk/cosmos/azure-cosmos-tests/pom.xml

@@ -252,7 +252,7 @@ Licensed under the MIT License.
               <value>com.azure.cosmos.CosmosNettyLeakDetectorFactory</value>
             </property>
           </properties>
-          <skipTests>true</skipTests>
+          <skipTests>false</skipTests>
         </configuration>
       </plugin>
 
@@ -934,3 +934,6 @@ Licensed under the MIT License.
     </profile>
   </profiles>
 </project>
+
+
+

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/implementation/http/Http2ParentChannelExceptionHandlerTest.java

@@ -0,0 +1,110 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.http;
+
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * Verifies that {@link Http2ParentChannelExceptionHandler} uses exception type
+ * and connection state to determine suppression behavior.
+ *
+ * Uses Netty's {@link EmbeddedChannel} which records unhandled exceptions internally.
+ * {@code checkException()} re-throws any exception that reached the pipeline tail.
+ *
+ * Note: EmbeddedChannel does not have an Http2FrameCodec in its pipeline, so
+ * {@code getActiveStreamCount()} returns -1. The handler treats -1 as "unknown"
+ * which takes the WARN path (safe default). The idle-connection DEBUG path
+ * requires both activeStreams == 0 AND !channelActive, which we simulate by
+ * closing the channel before firing the exception.
+ */
+public class Http2ParentChannelExceptionHandlerTest {
+
+    /**
+     * BEFORE fix — without the handler, exceptions reach the pipeline tail.
+     * EmbeddedChannel's checkException() re-throws the unhandled exception,
+     * proving it reached Netty's TailContext (which in production logs as WARN).
+     */
+    @Test(groups = "unit")
+    public void withoutHandler_exceptionReachesTail() {
+        EmbeddedChannel channel = new EmbeddedChannel();
+
+        channel.pipeline().fireExceptionCaught(
+            new IOException("Connection reset by peer"));
+
+        assertThatThrownBy(channel::checkException)
+            .isInstanceOf(IOException.class)
+            .hasMessageContaining("Connection reset by peer");
+
+        channel.finishAndReleaseAll();
+    }
+
+    /**
+     * IOException on active channel (no Http2FrameCodec → activeStreams = -1)
+     * is consumed by the handler (logged at WARN internally, since activeStreams
+     * is unknown) but does NOT propagate to TailContext. Channel stays open.
+     */
+    @Test(groups = "unit")
+    public void withHandler_ioExceptionOnActiveChannel_consumedAndChannelStaysOpen() {
+        EmbeddedChannel channel = new EmbeddedChannel(
+            new Http2ParentChannelExceptionHandler());
+
+        assertThat(channel.isActive()).isTrue();
+
+        channel.pipeline().fireExceptionCaught(
+            new IOException("Connection reset by peer"));
+
+        // Exception consumed — does NOT reach tail
+        channel.checkException();
+
+        // Channel is NOT closed — handler does not alter lifecycle
+        assertThat(channel.isOpen()).isTrue();
+
+        channel.finishAndReleaseAll();
+    }
+
+    /**
+     * Non-IO exception (e.g., RuntimeException, NPE) is NOT consumed —
+     * it propagates to TailContext so it surfaces as WARN in production.
+     * This ensures we don't swallow unexpected/unknown exceptions.
+     */
+    @Test(groups = "unit")
+    public void withHandler_nonIoException_propagatesToTail() {
+        EmbeddedChannel channel = new EmbeddedChannel(
+            new Http2ParentChannelExceptionHandler());
+
+        channel.pipeline().fireExceptionCaught(
+            new RuntimeException("Unexpected state error"));
+
+        // Exception propagated — reaches tail
+        assertThatThrownBy(channel::checkException)
+            .isInstanceOf(RuntimeException.class)
+            .hasMessageContaining("Unexpected state error");
+
+        channel.finishAndReleaseAll();
+    }
+
+    /**
+     * NullPointerException is also propagated — we only suppress IOException.
+     */
+    @Test(groups = "unit")
+    public void withHandler_nullPointerException_propagatesToTail() {
+        EmbeddedChannel channel = new EmbeddedChannel(
+            new Http2ParentChannelExceptionHandler());
+
+        channel.pipeline().fireExceptionCaught(
+            new NullPointerException("handler bug"));
+
+        assertThatThrownBy(channel::checkException)
+            .isInstanceOf(NullPointerException.class)
+            .hasMessageContaining("handler bug");
+
+        channel.finishAndReleaseAll();
+    }
+}

sdk/cosmos/azure-cosmos/CHANGELOG.md

@@ -12,6 +12,7 @@
 * Fixed JVM `<clinit>` deadlock when multiple threads concurrently trigger Cosmos SDK class loading for the first time. - See [PR 48689](https://github.com/Azure/azure-sdk-for-java/pull/48689)
 * Fixed an issue where `CustomItemSerializer` was incorrectly applied to internal SDK query pipeline structures (e.g., `OrderByRowResult`, `Document`), causing deserialization failures in ORDER BY, GROUP BY, aggregate, DISTINCT, and hybrid search queries. - See [PR 48811](https://github.com/Azure/azure-sdk-for-java/pull/48811)
 * Fixed an issue where `SqlParameter` ignored the configured `CustomItemSerializer`, always using the internal default serializer instead. - See [PR 48811](https://github.com/Azure/azure-sdk-for-java/pull/48811)
+* Fixed Netty WARN log "An exceptionCaught() event was fired, and it reached at the tail of the pipeline" appearing on HTTP/2 connections when the server resets idle TCP connections. Added an exception handler on the HTTP/2 parent channel to consume connection-level exceptions at DEBUG level, matching HTTP/1.1 behavior. - See [PR 48687](https://github.com/Azure/azure-sdk-for-java/pull/48687)
 
 #### Other Changes
 

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/Http2ParentChannelExceptionHandler.java

@@ -0,0 +1,83 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+package com.azure.cosmos.implementation.http;
+
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.handler.codec.http2.Http2FrameCodec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Exception handler for the HTTP/2 parent (TCP) channel pipeline.
+ * <p>
+ * In HTTP/2, reactor-netty multiplexes streams on a shared parent TCP connection.
+ * Child stream channels have {@code ChannelOperationsHandler} which catches exceptions
+ * and fails the active subscriber (matching HTTP/1.1 behavior). However, the parent
+ * channel has no such handler — exceptions propagate to Netty's {@code TailContext}
+ * which logs them as WARN ("An exceptionCaught() event was fired, and it reached at
+ * the tail of the pipeline").
+ * <p>
+ * This handler uses two dimensions to decide how to handle exceptions:
+ * <ol>
+ *   <li><b>Exception type</b> — only {@link IOException} is suppressed. Non-IO exceptions
+ *       (bugs, unexpected state) are propagated so they surface via Netty's TailContext WARN.</li>
+ *   <li><b>Connection state</b> — for IO exceptions, the handler checks the number of active
+ *       HTTP/2 streams and whether the channel is still active. An IO exception on an idle,
+ *       inactive connection (0 streams, channel closed) is logged at DEBUG. An IO exception
+ *       with active streams or on a still-active channel is logged at WARN to preserve
+ *       telemetry for connection disruptions that affect in-flight requests.</li>
+ * </ol>
+ * <p>
+ * The handler does NOT close the channel or alter connection lifecycle — reactor-netty
+ * and the connection pool's eviction predicate ({@code !channel.isActive()}) handle that
+ * independently.
+ *
+ * @see ReactorNettyClient#configureChannelPipelineHandlers()
+ */
+final class Http2ParentChannelExceptionHandler extends ChannelInboundHandlerAdapter {
+
+    static final String HANDLER_NAME = "cosmosH2ParentExceptionHandler";
+
+    private static final Logger logger = LoggerFactory.getLogger(Http2ParentChannelExceptionHandler.class);
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
+        if (cause instanceof IOException) {
+            int activeStreams = getActiveStreamCount(ctx);
+            boolean channelActive = ctx.channel().isActive();
+
+            if (activeStreams == 0 && !channelActive) {
+                // Idle connection died (e.g., TCP RST from LB idle timeout) — no impact.
+                if (logger.isDebugEnabled()) {
+                    logger.debug(
+                        "Exception on idle HTTP/2 parent connection [id:{}, activeStreams=0, channelActive=false]: {}",
+                        ctx.channel().id().asShortText(), cause.toString(), cause);
+                }
+            } else {
+                // IO exception with live streams or channel still active — worth alerting.
+                logger.warn(
+                    "Exception on HTTP/2 parent connection [id:{}, activeStreams={}, channelActive={}]: {}",
+                    ctx.channel().id().asShortText(), activeStreams, channelActive, cause.toString(), cause);
+            }
+        } else {
+            // Non-IO exception — propagate to TailContext so it surfaces as WARN.
+            ctx.fireExceptionCaught(cause);
+        }
+    }
+
+    private static int getActiveStreamCount(ChannelHandlerContext ctx) {
+        try {
+            Http2FrameCodec codec = ctx.pipeline().get(Http2FrameCodec.class);
+            if (codec != null) {
+                return codec.connection().numActiveStreams();
+            }
+        } catch (Exception ignored) {
+            // Codec not available or connection already torn down
+        }
+        return -1;
+    }
+}

sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/http/ReactorNettyClient.java

@@ -166,6 +166,25 @@ private void configureChannelPipelineHandlers() {
                             "customHeaderCleaner",
                             new Http2ResponseHeaderCleanerHandler());
                     }
+
+                    // Install exception handler on the HTTP/2 parent (TCP) channel.
+                    // In H2, doOnConnected fires for stream (child) channels — channel.parent()
+                    // is the TCP connection. The parent pipeline has no ChannelOperationsHandler
+                    // (unlike H1.1), so TCP-level exceptions (RST, broken pipe) propagate to
+                    // Netty's TailContext and get logged as WARN. This handler matches H1.1
+                    // behavior by consuming exceptions at DEBUG level.
+                    Channel parent = connection.channel().parent();
+                    if (parent != null
+                        && parent.pipeline().get(Http2ParentChannelExceptionHandler.HANDLER_NAME) == null) {
+
+                        try {
+                            parent.pipeline().addLast(
+                                Http2ParentChannelExceptionHandler.HANDLER_NAME,
+                                new Http2ParentChannelExceptionHandler());
+                        } catch (IllegalArgumentException ignored) {
+                            // Duplicate handler — already installed by a concurrent stream
+                        }
+                    }
                 }));
         }
     }