[6.1] Test | Fix Transient Fault handling and other flaky unit tests (#4080)

eng/pipelines/common/templates/steps/configure-sql-server-macos-step.yml

@@ -4,8 +4,8 @@
 # See the LICENSE file in the project root for more information.                #
 #################################################################################
 
-# This step installs the latest SQL Server 2022 onto the macOS host and
-# configures it for use.
+# This step installs the latest SQL Server 2022 onto the macOS host as a docker
+# container and configures it for use.
 
 parameters:
   - name: password
@@ -21,39 +21,55 @@ steps:
 - bash: |
     # The "user" pipeline variable conflicts with homebrew, causing errors during install. Set it back to the pipeline user.
     USER=`whoami`
+
     SQLCMD_ERRORS=$(Agent.TempDirectory)/sqlcmd_err.log
-    echo $SQLCMD_ERRORS
+    echo "Errors will be written to: $SQLCMD_ERRORS"
+
+    # Configure the prompt to show the current timestamp so we can see how long each command takes.
+    export PS4='+ [$(date "+%Y-%m-%d %H:%M:%S")] '
+    set -x
 
+    # Install Docker CLI (not Desktop -- Colima provides the daemon) and SQLCMD tools.
     brew install colima
-    brew install --cask docker
+    brew install docker
     brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release
     brew update
     HOMEBREW_ACCEPT_EULA=Y brew install mssql-tools18
-    colima start --arch x86_64
+
+    # Start Colima with Virtualization.framework for x86_64 binary translation
+    # on Apple Silicon.  Rosetta/binfmt emulation is enabled by default in
+    # Colima >= 0.8 when using --vm-type vz, which is dramatically faster than
+    # --arch x86_64 (full QEMU VM emulation).
+    # Requires macOS >= 13 (Ventura).
+    colima start --vm-type vz --cpu 4 --memory 4
     docker --version
-    docker pull mcr.microsoft.com/mssql/server:2022-latest
-
+    docker pull --platform linux/amd64 mcr.microsoft.com/mssql/server:2022-latest
+
+    # Disable xtrace before any command that handles the SA password so it is
+    # not echoed (expanded) into pipeline logs.
+    { set +x; } 2>/dev/null
+
     # Password for the SA user (required)
-    MSSQL_SA_PW=${{ parameters.password }}
+    MSSQL_SA_PW="${{ parameters.password }}"
+
+    docker run --platform linux/amd64 -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest
 
-    docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PW" -p 1433:1433 -p 1434:1434 --name sql1 --hostname sql1 -d mcr.microsoft.com/mssql/server:2022-latest
-
-    sleep 5
+    sleep 10
 
     docker ps -a
 
     # Connect to the SQL Server container and get its version.
     #
-    # It can take a while for the docker container to start listening and be
-    # ready for connections, so we will wait for up to 2 minutes, checking every
-    # 3 seconds.
-
-    # Wait 3 seconds between attempts.
-    delay=3
-    
-    # Try up to 40 times (2 minutes) to connect.
-    maxAttempts=40
-    
+    # With Rosetta 2 emulation, SQL Server starts much faster than under full
+    # QEMU emulation, but it can still take a minute or two.  We allow up to
+    # 6 minutes (72 attempts x 5 seconds) as a generous upper bound.
+
+    # Wait 5 seconds between attempts.
+    delay=5
+
+    # Try up to 72 times (~6 minutes) to connect.
+    maxAttempts=72
+
     # Attempt counter.
     attempt=1
 
@@ -63,18 +79,29 @@ steps:
     while [ $attempt -le $maxAttempts ]
     do
       echo "Waiting for SQL Server to start (attempt #$attempt of $maxAttempts)..."
-
-      sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1
-
+
+      # -C trusts the self-signed certificate inside the container.
+      sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" >> $SQLCMD_ERRORS 2>&1
+
       # If the command was successful, then the SQL Server is ready.
       if [ $? -eq 0 ]; then
         ready=1
         break
       fi
-
+
+      # Verify the container is still running; no point retrying if it crashed.
+      if ! docker ps --filter "name=^/sql1$" --filter "status=running" --format '{{.Names}}' | grep -Fxq 'sql1'; then
+        echo "ERROR: sql1 container is no longer running."
+        docker ps -a --filter "name=^/sql1$"
+        echo "--- Container logs ---"
+        docker logs sql1 2>&1 | tail -50
+        rm -f $SQLCMD_ERRORS
+        exit 1
+      fi
+
       # Increment the attempt counter.
       ((attempt++))
-    
+
       # Wait before trying again.
       sleep $delay
     done
@@ -83,8 +110,13 @@ steps:
     if [ $ready -eq 0 ]
     then
       # No, so report the error(s) and exit.
-      echo Cannot connect to SQL Server; installation aborted; errors were:
+      echo "Cannot connect to SQL Server after $maxAttempts attempts; installation aborted."
+      echo "--- sqlcmd errors ---"
       cat $SQLCMD_ERRORS
+      echo "--- Container status ---"
+      docker ps -a --filter "name=^/sql1$"
+      echo "--- Container logs (last 80 lines) ---"
+      docker logs sql1 2>&1 | tail -80
       rm -f $SQLCMD_ERRORS
       exit 1
     fi
@@ -93,22 +125,22 @@ steps:
 
     echo "Use sqlcmd to show which IP addresses are being listened on..."
     echo 0.0.0.0
-    sqlcmd -S 0.0.0.0 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2
+    sqlcmd -S 0.0.0.0 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2
     echo 127.0.0.1
-    sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2
+    sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2
     echo ::1
-    sqlcmd -S ::1 -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2
+    sqlcmd -S ::1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2
     echo localhost
-    sqlcmd -S localhost -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2
+    sqlcmd -S localhost -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2
     echo "(sqlcmd default / not specified)"
-    sqlcmd -No -U sa -P $MSSQL_SA_PW -Q "SELECT @@VERSION" -l 2
+    sqlcmd -No -C -U sa -P "$MSSQL_SA_PW" -Q "SELECT @@VERSION" -l 2
 
     echo "Configuring Dedicated Administer Connections to allow remote connections..."
-    sqlcmd -S 127.0.0.1 -No -U sa -P $MSSQL_SA_PW -Q "sp_configure 'remote admin connections', 1; RECONFIGURE;"
+    sqlcmd -S 127.0.0.1 -No -C -U sa -P "$MSSQL_SA_PW" -Q "sp_configure 'remote admin connections', 1; RECONFIGURE;"
     if [ $? = 1 ]
     then
       echo "Error configuring DAC for remote access."
-      exit $errstatus
+      exit 1
     else
       echo "Configuration complete."
     fi

src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlTypes/SqlTypeWorkaroundsTests.cs

@@ -63,7 +63,7 @@ public void SqlBinaryCtor_NullInput()
             };
 
         [Theory]
-        [MemberData(nameof(SqlDecimalExtractData_NonNullInput_Data))]
+        [MemberData(nameof(SqlDecimalExtractData_NonNullInput_Data), DisableDiscoveryEnumeration = true)]
         public void SqlDecimalExtractData_NonNullInput(SqlDecimal input)
         {
             // Act
@@ -156,7 +156,7 @@ public void SqlGuidCtor_ValidInput(byte[] input)
             };
 
         [Theory]
-        [MemberData(nameof(SqlMoneyCtor_Data))]
+        [MemberData(nameof(SqlMoneyCtor_Data), DisableDiscoveryEnumeration = true)]
         public void SqlMoneyCtor(long input, SqlMoney expected)
         {
             // Act
@@ -177,7 +177,7 @@ public void SqlMoneyCtor(long input, SqlMoney expected)
             };
 
         [Theory]
-        [MemberData(nameof(SqlMoneyToSqlInternalRepresentation_NonNullInput_Data))]
+        [MemberData(nameof(SqlMoneyToSqlInternalRepresentation_NonNullInput_Data), DisableDiscoveryEnumeration = true)]
         public void SqlMoneyToSqlInternalRepresentation_NonNullInput(SqlMoney input, long expected)
         {
             // Act

src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionFailoverTests.cs

@@ -11,7 +11,6 @@
 
 namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests
 {
-    [Trait("Category", "flaky")]
     [Collection("SimulatedServerTests")]
     public class ConnectionFailoverTests
     {
@@ -71,7 +70,7 @@ public void TransientFault_NoFailover_DoesNotClearPool(uint errorCode)
             Assert.Equal($"localhost,{initialServer.EndPoint.Port}", secondConnection.DataSource);
 
             // 1 for the initial connection, 2 for the second connection
-            Assert.Equal(3, initialServer.PreLoginCount);
+            Assert.Equal(3, initialServer.Login7Count);
             // A failover should not be triggered, so prelogin count to the failover server should be 0
             Assert.Equal(0, failoverServer.PreLoginCount);
         }
@@ -219,6 +218,7 @@ public void NetworkDelay_ShouldConnectToPrimary()
                 InitialCatalog = "master",// Required for failover partner to work
                 ConnectTimeout = 5,
                 Encrypt = false,
+                Pooling = false, // Disable pooling to ensure a fresh connection attempt is made
                 MultiSubnetFailover = false,
 #if NETFRAMEWORK
                 TransparentNetworkIPResolution = false,
@@ -275,6 +275,7 @@ public void NetworkError_WithUserProvidedPartner_RetryDisabled_ShouldConnectToFa
                 ConnectRetryCount = 0, // Disable retry
                 FailoverPartner = $"localhost,{failoverServer.EndPoint.Port}", // User provided failover partner
                 Encrypt = false,
+                Pooling = false, // Disable pooling to ensure a fresh connection attempt is made on failover
             };
             using SqlConnection connection = new(builder.ConnectionString);
             try
@@ -326,6 +327,9 @@ public void NetworkError_WithUserProvidedPartner_RetryEnabled_ShouldConnectToFai
                 ConnectRetryInterval = 1,
                 FailoverPartner = $"localhost,{failoverServer.EndPoint.Port}", // User provided failover partner
                 Encrypt = false,
+#if NETFRAMEWORK
+                TransparentNetworkIPResolution = false,
+#endif
             };
             using SqlConnection connection = new(builder.ConnectionString);
             // Act
@@ -337,7 +341,11 @@ public void NetworkError_WithUserProvidedPartner_RetryEnabled_ShouldConnectToFai
             Assert.Equal(ConnectionState.Open, connection.State);
             Assert.Equal($"localhost,{failoverServer.EndPoint.Port}", connection.DataSource);
             Assert.Equal(1, server.PreLoginCount);
-            Assert.Equal(1, failoverServer.PreLoginCount);
+            // Login7 is sent to the primary but the client gives up during the
+            // server-side delay; the counter is still incremented when the
+            // Login7 message is received.
+            Assert.Equal(1, server.Login7Count);
+            Assert.Equal(1, failoverServer.Login7Count);
         }
 
         [Theory]
@@ -370,7 +378,8 @@ public void TransientFault_ShouldConnectToPrimary(uint errorCode)
                 InitialCatalog = "master",
                 ConnectTimeout = 30,
                 ConnectRetryInterval = 1,
-                Encrypt = false
+                Encrypt = false,
+                Pooling = false, // Disable pooling to ensure a fresh connection attempt is made
             };
             using SqlConnection connection = new(builder.ConnectionString);
 
@@ -382,7 +391,7 @@ public void TransientFault_ShouldConnectToPrimary(uint errorCode)
             Assert.Equal($"localhost,{server.EndPoint.Port}", connection.DataSource);
 
             // Failures should prompt the client to return to the original server, resulting in a login count of 2
-            Assert.Equal(2, server.PreLoginCount);
+            Assert.Equal(2, server.Login7Count);
         }
 
         [Theory]
@@ -468,7 +477,7 @@ public void TransientFault_WithUserProvidedPartner_ShouldConnectToPrimary(uint e
                 FailoverPartner = $"localhost:{failoverServer.EndPoint.Port}", // User provided failover partner
             };
             using SqlConnection connection = new(builder.ConnectionString);
-                
+
             // Act
             connection.Open();
 
@@ -477,7 +486,7 @@ public void TransientFault_WithUserProvidedPartner_ShouldConnectToPrimary(uint e
             Assert.Equal($"localhost,{server.EndPoint.Port}", connection.DataSource);
 
             // Failures should prompt the client to return to the original server, resulting in a login count of 2
-            Assert.Equal(2, server.PreLoginCount);
+            Assert.Equal(2, server.Login7Count);
         }
 
         [Theory]
@@ -580,6 +589,10 @@ public void TransientFault_IgnoreServerProvidedFailoverPartner_ShouldConnectToUs
             // Dispose of the server to trigger a failover
             server.Dispose();
 
+            // Clear the pool to ensure the next connection attempt doesn't reuse
+            // the pooled connection to the now-disposed primary server.
+            SqlConnection.ClearAllPools();
+
             // Opening a new connection will use the failover partner stored in the pool group.
             // This will fail if the server provided failover partner was stored to the pool group.
             using SqlConnection failoverConnection = new(builder.ConnectionString);
@@ -593,9 +606,9 @@ public void TransientFault_IgnoreServerProvidedFailoverPartner_ShouldConnectToUs
             Assert.Equal(ConnectionState.Open, failoverConnection.State);
             Assert.Equal($"localhost,{failoverServer.EndPoint.Port}", failoverConnection.DataSource);
             // 1 for the initial connection
-            Assert.Equal(1, server.PreLoginCount);
+            Assert.Equal(1, server.Login7Count);
             // 1 for the failover connection
-            Assert.Equal(1, failoverServer.PreLoginCount);
+            Assert.Equal(1, failoverServer.Login7Count);
         }
     }
 }

src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTests.cs

@@ -10,7 +10,6 @@
 
 namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests
 {
-    [Trait("Category", "flaky")]
     [Collection("SimulatedServerTests")]
     public class ConnectionRoutingTests
     {
@@ -58,8 +57,8 @@ public void TransientFaultAtRoutedLocation_ShouldReturnToGateway(uint errorCode)
             Assert.Equal($"localhost,{router.EndPoint.Port}", connection.DataSource);
 
             // Failures should prompt the client to return to the original server, resulting in a login count of 2
-            Assert.Equal(2, router.PreLoginCount);
-            Assert.Equal(2, server.PreLoginCount);
+            Assert.Equal(2, router.Login7Count);
+            Assert.Equal(2, server.Login7Count);
         }
 
         [Theory]

src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/ConnectionRoutingTestsAzure.cs

@@ -10,7 +10,6 @@
 
 namespace Microsoft.Data.SqlClient.UnitTests.SimulatedServerTests
 {
-    [Trait("Category", "flaky")]
     [Collection("SimulatedServerTests")]
     public class ConnectionRoutingTestsAzure : IDisposable
     {
@@ -76,8 +75,8 @@ public void TransientFaultAtRoutedLocation_ShouldReturnToGateway(uint errorCode)
             Assert.Equal($"localhost,{router.EndPoint.Port}", connection.DataSource);
 
             // Failures should prompt the client to return to the original server, resulting in a login count of 2
-            Assert.Equal(2, router.PreLoginCount);
-            Assert.Equal(2, server.PreLoginCount);
+            Assert.Equal(2, router.Login7Count);
+            Assert.Equal(2, server.Login7Count);
         }
 
         [Theory]