Implement pool shutdown for ChannelDbConnectionPool and harden WaitHandleDbConnectionPool shutdown (AB#44828)

ChannelDbConnectionPool:
- Shutdown() now transitions State to ShuttingDown (FR-001), completes the idle
  channel writer to unblock async waiters (FR-002, FR-007), and drains and
  destroys buffered idle connections (FR-003). Implementation is idempotent via
  Interlocked.CompareExchange (FR-006).
- Startup() emits a trace and is a structural counterpart to Shutdown.
- ReturnInternalConnection no longer asserts on TryWrite; if the channel was
  completed by a concurrent shutdown, the returning connection is destroyed
  (FR-004 race-window guard).
- IdleConnectionChannel.Complete() exposes channel completion to the pool.

WaitHandleDbConnectionPool:
- Shutdown() is now idempotent (FR-006), disposes both _cleanupTimer and
  _errorTimer (FR-005), wakes threads parked in WaitHandle.WaitAny by releasing
  the pool semaphore (FR-007), and drains _stackNew/_stackOld (FR-003).
- CleanupCallback and ErrorCallback short-circuit when State == ShuttingDown so
  an in-flight callback racing with Shutdown does not re-arm work.
- The private TryGetConnection wait loop re-checks State after WaitHandle.WaitAny
  and bails out, so waiters cannot spin against a drained pool.

Tests:
- ChannelDbConnectionPoolShutdownTest (7 tests) covers state transition, drain,
  return-after-shutdown, idempotency, async waiter unblock, post-shutdown return,
  and Startup-after-Shutdown.
- WaitHandleDbConnectionPoolShutdownTest (9 tests) covers state transition,
  cleanup/error timer disposal, drain, idempotency, callback no-op after
  shutdown, sync TryGetConnection short-circuit, and sync waiter unblock.

Verified by running targeted suite: 340 tests passing across net8.0, net9.0,
net10.0, and net462.

Spec: specs/004-pool-shutdown.

Author: priyankatiwari08

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/ChannelDbConnectionPool.cs

@@ -92,6 +92,13 @@ internal sealed class ChannelDbConnectionPool : IDbConnectionPool
         /// Must be updated using <see cref="Interlocked"/> operations to ensure thread safety.
         /// </summary>
         private volatile int _isClearing;
+
+        /// <summary>
+        /// Tracks whether <see cref="Shutdown"/> has already initiated the shutdown sequence so that
+        /// repeated calls are observed as no-ops (FR-006). Updated atomically via
+        /// <see cref="Interlocked.CompareExchange(ref int, int, int)"/>.
+        /// </summary>
+        private int _shutdownInitiated;
         #endregion
 
         /// <summary>
@@ -254,21 +261,57 @@ public void ReturnInternalConnection(DbConnectionInternal connection, DbConnecti
             }
             else
             {
-                var written = _idleChannel.TryWrite(connection);
-                Debug.Assert(written, "Failed to write returning connection to the idle channel.");
+                if (!_idleChannel.TryWrite(connection))
+                {
+                    // The channel has been completed (pool is shutting down). Race window
+                    // between the State check above and TryWrite: destroy instead of pooling.
+                    RemoveConnection(connection);
+                }
             }
         }
 
         /// <inheritdoc />
         public void Shutdown()
         {
-            // No-op for now, warmup will be implemented later.
+            // FR-006: idempotent. Compare-and-exchange ensures only one caller performs shutdown work.
+            if (Interlocked.CompareExchange(ref _shutdownInitiated, 1, 0) != 0)
+            {
+                return;
+            }
+
+            SqlClientEventSource.Log.TryPoolerTraceEvent(
+                "<prov.DbConnectionPool.Shutdown|RES|INFO|CPOOL> {0}", Id);
+
+            // FR-001: transition to ShuttingDown. After this point, ReturnInternalConnection
+            // routes returning connections to RemoveConnection.
+            State = ShuttingDown;
+
+            // FR-002 + FR-007: complete the channel writer so:
+            //  - no further idle connections can be enqueued (TryWrite returns false), and
+            //  - in-flight / future async waiters on ReadAsync fault with ChannelClosedException.
+            _idleChannel.Complete();
+
+            // FR-003: drain remaining buffered idle connections and destroy them. The channel is
+            // unbounded so all already-enqueued items can be drained synchronously.
+            while (_idleChannel.TryRead(out DbConnectionInternal? connection))
+            {
+                if (connection is not null)
+                {
+                    RemoveConnection(connection);
+                }
+                // null sentinels are wake-up signals only; nothing to destroy.
+            }
         }
 
         /// <inheritdoc />
         public void Startup()
         {
-            // No-op for now, warmup will be implemented later.
+            // This pool has no background timers today (idle timeout is enforced lazily in
+            // IsLiveConnection on retrieval; pruning is not implemented). State is set to Running
+            // in the constructor, so this is currently the symmetrical counterpart of Shutdown.
+            // Background work (warmup, pruning timers) will be added here when introduced.
+            SqlClientEventSource.Log.TryPoolerTraceEvent(
+                "<prov.DbConnectionPool.Startup|RES|INFO|CPOOL> {0}", Id);
         }
 
         /// <inheritdoc />

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/IdleConnectionChannel.cs

@@ -26,10 +26,19 @@ internal IdleConnectionChannel()
         {
             var channel = Channel.CreateUnbounded<DbConnectionInternal?>();
             _reader = channel.Reader;
-            //TODO: the channel should be completed on pool shutdown
             _writer = channel.Writer;
         }
 
+        /// <summary>
+        /// Marks the channel writer as complete. After completion, <see cref="TryWrite"/>
+        /// returns <see langword="false"/> for any future writes, and any in-flight or future
+        /// <see cref="ReadAsync"/> waiters will fault with <see cref="System.Threading.Channels.ChannelClosedException"/>
+        /// once the channel is drained. Used by the connection pool to signal shutdown.
+        /// </summary>
+        /// <returns><see langword="true"/> if this call completed the channel; otherwise <see langword="false"/>
+        /// (channel was already completed).</returns>
+        internal bool Complete() => _writer.TryComplete();
+
         /// <summary>
         /// The number of non-null connections currently in the channel.
         /// </summary>

src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ConnectionPool/WaitHandleDbConnectionPool.cs

@@ -329,6 +329,14 @@ public bool IsRunning
 
         private void CleanupCallback(object state)
         {
+            // FR-005: if the pool is shutting down, skip work. Shutdown disposes the timer, but
+            // a callback may already be in-flight when Shutdown runs; this guard ensures it does
+            // not perform pruning or re-arm pool create requests.
+            if (State == ShuttingDown)
+            {
+                return;
+            }
+
             // Called when the cleanup-timer ticks over.
 
             // This is the automatic pruning method.  Every period, we will
@@ -767,6 +775,13 @@ private void DestroyObject(DbConnectionInternal obj)
 
         private void ErrorCallback(object state)
         {
+            // FR-005: skip work if the pool is shutting down. The shutdown path disposes the
+            // timer; this guard handles the in-flight-callback race.
+            if (State == ShuttingDown)
+            {
+                return;
+            }
+
             SqlClientEventSource.Log.TryPoolerTraceEvent("<prov.DbConnectionPool.ErrorCallback|RES|CPOOL> {0}, Resetting Error handling.", Id);
             _errorOccurred = false;
             _waitHandles.ErrorEvent.Reset();
@@ -956,6 +971,16 @@ private bool TryGetConnection(DbConnection owningObject, uint waitForMultipleObj
                     {
                         waitResult = WaitHandle.WaitAny(_waitHandles.GetHandles(allowCreate), unchecked((int)waitForMultipleObjectsTimeout));
 
+                        // FR-007: after waking, observe shutdown state and bail out so waiters
+                        // do not spin against a drained pool.
+                        if (State != Running)
+                        {
+                            SqlClientEventSource.Log.TryPoolerTraceEvent("<prov.DbConnectionPool.GetConnection|RES|CPOOL> {0}, Pool is shutting down; abandoning wait.", Id);
+                            Interlocked.Decrement(ref _waitCount);
+                            connection = null;
+                            return false;
+                        }
+
                         // From the WaitAny docs: "If more than one object became signaled during
                         // the call, this is the array index of the signaled object with the
                         // smallest index value of all the signaled objects."  This is important
@@ -1481,14 +1506,45 @@ public void Startup()
         public void Shutdown()
         {
             SqlClientEventSource.Log.TryPoolerTraceEvent("<prov.DbConnectionPool.Shutdown|RES|INFO|CPOOL> {0}", Id);
+
+            // FR-006: idempotent. Subsequent calls observe ShuttingDown and bail.
+            if (State == ShuttingDown)
+            {
+                return;
+            }
             State = ShuttingDown;
 
-            // deactivate timer callbacks
-            Timer t = _cleanupTimer;
-            _cleanupTimer = null;
-            if (t != null)
+            // FR-005: dispose all background timers so they no longer schedule new work.
+            // Note that any timer callback already in flight may still observe State == ShuttingDown
+            // and short-circuit (see CleanupCallback / ErrorCallback).
+            Timer cleanup = Interlocked.Exchange(ref _cleanupTimer, null);
+            cleanup?.Dispose();
+            Timer error = Interlocked.Exchange(ref _errorTimer, null);
+            error?.Dispose();
+
+            // FR-007: wake any threads parked in WaitHandle.WaitAny by releasing the pool semaphore
+            // up to its max count. Waiters check State == Running after wake-up and bail.
+            // We may release more than necessary; SemaphoreFullException is harmless when the
+            // semaphore is already saturated.
+            try
+            {
+                _waitHandles.PoolSemaphore.Release(MaxPoolSize);
+            }
+            catch (SemaphoreFullException)
+            {
+                // Already at max count - all slots free. Nothing to do.
+            }
+
+            // FR-003: drain idle stacks and destroy contained connections. Active checked-out
+            // connections are destroyed when they are returned (see DeactivateObject's
+            // State is ShuttingDown branch).
+            while (_stackNew.TryPop(out DbConnectionInternal newObj))
+            {
+                DestroyObject(newObj);
+            }
+            while (_stackOld.TryPop(out DbConnectionInternal oldObj))
             {
-                t.Dispose();
+                DestroyObject(oldObj);
             }
         }
 

src/Microsoft.Data.SqlClient/tests/UnitTests/ConnectionPool/ChannelDbConnectionPoolShutdownTest.cs

@@ -0,0 +1,190 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Data.Common;
+using System.Threading.Tasks;
+using Microsoft.Data.Common.ConnectionString;
+using Microsoft.Data.ProviderBase;
+using Microsoft.Data.SqlClient.ConnectionPool;
+using Xunit;
+
+namespace Microsoft.Data.SqlClient.UnitTests.ConnectionPool
+{
+    /// <summary>
+    /// Deterministic tests for <see cref="ChannelDbConnectionPool"/> shutdown behavior.
+    /// Verifies the contract defined by spec 004-pool-shutdown FR-001 .. FR-007 (P1 scope).
+    /// </summary>
+    public class ChannelDbConnectionPoolShutdownTest
+    {
+        private static ChannelDbConnectionPool ConstructPool(int maxPoolSize = 5)
+        {
+            var poolGroupOptions = new DbConnectionPoolGroupOptions(
+                poolByIdentity: false,
+                minPoolSize: 0,
+                maxPoolSize: maxPoolSize,
+                creationTimeout: 15,
+                loadBalanceTimeout: 0,
+                hasTransactionAffinity: true);
+
+            var dbConnectionPoolGroup = new DbConnectionPoolGroup(
+                new SqlConnectionOptions("Data Source=localhost;"),
+                new ConnectionPoolKey("TestDataSource", credential: null, accessToken: null, accessTokenCallback: null, sspiContextProvider: null),
+                poolGroupOptions);
+
+            return new ChannelDbConnectionPool(
+                new ChannelDbConnectionPoolTest.SuccessfulSqlConnectionFactory(),
+                dbConnectionPoolGroup,
+                DbConnectionPoolIdentity.NoIdentity,
+                new DbConnectionPoolProviderInfo());
+        }
+
+        // FR-001
+        [Fact]
+        public void Shutdown_TransitionsState_ToShuttingDown()
+        {
+            var pool = ConstructPool();
+            Assert.True(pool.IsRunning);
+            Assert.Equal(DbConnectionPoolState.Running, pool.State);
+
+            pool.Shutdown();
+
+            Assert.False(pool.IsRunning);
+            Assert.Equal(DbConnectionPoolState.ShuttingDown, pool.State);
+        }
+
+        // FR-003
+        [Fact]
+        public void Shutdown_DrainsIdleConnections()
+        {
+            var pool = ConstructPool();
+
+            // Vend and return three connections so they sit idle in the channel.
+            var owners = new List<SqlConnection>();
+            var conns = new List<DbConnectionInternal>();
+            for (int i = 0; i < 3; i++)
+            {
+                var owner = new SqlConnection();
+                owners.Add(owner);
+                Assert.True(pool.TryGetConnection(owner, taskCompletionSource: null, out DbConnectionInternal? c));
+                Assert.NotNull(c);
+                conns.Add(c!);
+            }
+            for (int i = 0; i < conns.Count; i++)
+            {
+                pool.ReturnInternalConnection(conns[i], owners[i]);
+            }
+            Assert.Equal(3, pool.IdleCount);
+
+            pool.Shutdown();
+
+            Assert.Equal(0, pool.IdleCount);
+            Assert.Equal(0, pool.Count);
+        }
+
+        // FR-004 - returned connection while shutting down is destroyed, not pooled.
+        [Fact]
+        public void Shutdown_ReturnedConnection_IsDestroyedNotPooled()
+        {
+            var pool = ConstructPool();
+            var owner = new SqlConnection();
+            Assert.True(pool.TryGetConnection(owner, taskCompletionSource: null, out DbConnectionInternal? conn));
+            Assert.NotNull(conn);
+            Assert.Equal(1, pool.Count);
+            Assert.Equal(0, pool.IdleCount);
+
+            pool.Shutdown();
+
+            // Connection is still checked out; return it now.
+            pool.ReturnInternalConnection(conn!, owner);
+
+            Assert.Equal(0, pool.IdleCount);
+            Assert.Equal(0, pool.Count);
+        }
+
+        // FR-006
+        [Fact]
+        public void Shutdown_IsIdempotent()
+        {
+            var pool = ConstructPool();
+            pool.Shutdown();
+            // Second call must not throw and must leave state intact.
+            pool.Shutdown();
+            pool.Shutdown();
+            Assert.Equal(DbConnectionPoolState.ShuttingDown, pool.State);
+        }
+
+        // FR-007 - async waiter is unblocked when the pool shuts down.
+        [Fact]
+        public async Task Shutdown_UnblocksAsyncWaiter()
+        {
+            var pool = ConstructPool(maxPoolSize: 1);
+
+            // Saturate the pool.
+            Assert.True(pool.TryGetConnection(new SqlConnection(), taskCompletionSource: null, out DbConnectionInternal? blocking));
+            Assert.NotNull(blocking);
+
+            // Park an async waiter.
+            var tcs = new TaskCompletionSource<DbConnectionInternal>();
+            bool completed = pool.TryGetConnection(new SqlConnection(), tcs, out DbConnectionInternal? waiter);
+            Assert.False(completed);
+            Assert.Null(waiter);
+            Assert.False(tcs.Task.IsCompleted);
+
+            // Shut down the pool.
+            pool.Shutdown();
+
+            // Waiter must complete (faulted or with a null connection) within a bounded window.
+            var winner = await Task.WhenAny(tcs.Task, Task.Delay(TimeSpan.FromSeconds(5)));
+            Assert.Same(tcs.Task, winner);
+            // Either an exception was set (channel closed) or the result is null - both are acceptable
+            // shutdown signals. What matters is the waiter does NOT block forever.
+            if (tcs.Task.IsFaulted)
+            {
+                // Expected path: ChannelClosedException or a wrapped exception.
+                Assert.NotNull(tcs.Task.Exception);
+            }
+            else
+            {
+                // Permitted: completed with null result.
+                Assert.Null(tcs.Task.Result);
+            }
+        }
+
+        // Story 3 acceptance #3 - sync get fails fast after shutdown.
+        // The factory-level retry guard checks IsRunning, but the pool itself must not vend
+        // new connections after Shutdown. We verify by exhausting the pool first then
+        // checking that returned connections are destroyed (Count goes back to 0).
+        [Fact]
+        public void Shutdown_AfterShutdown_NewReturnsAreDestroyed()
+        {
+            var pool = ConstructPool();
+            var owner = new SqlConnection();
+            Assert.True(pool.TryGetConnection(owner, taskCompletionSource: null, out DbConnectionInternal? c));
+            Assert.NotNull(c);
+
+            pool.Shutdown();
+            pool.ReturnInternalConnection(c!, owner);
+
+            Assert.False(pool.IsRunning);
+            Assert.Equal(0, pool.Count);
+            Assert.Equal(0, pool.IdleCount);
+        }
+
+        // Sanity: Startup is currently a no-op for this pool but must not throw or change
+        // shutdown state if invoked after Shutdown.
+        [Fact]
+        public void Startup_AfterShutdown_DoesNotResurrectPool()
+        {
+            var pool = ConstructPool();
+            pool.Shutdown();
+
+            pool.Startup();
+
+            Assert.Equal(DbConnectionPoolState.ShuttingDown, pool.State);
+            Assert.False(pool.IsRunning);
+        }
+    }
+}