Fix intermittent hot-reload test failures caused by metadata initialization race condition (#3302)

## Why make this change?

Related to #2992

Some flakey tests were being masked by faster initialization when the
task they were in was isolated. When we refactored the tests to lower
the overall runtime in this PR
#3245 these flakey tests
were eventually revealed.

The root cause was a race condition: after a successful hot-reload,
`WaitForConditionAsync` detects the "Validated hot-reloaded
configuration file" console message and returns, but the engine's
metadata providers have not fully re-initialized yet. An immediate HTTP
request can arrive before the metadata is ready, causing a 500 error.

## What is this change?

Added retry logic to the three non-ignored tests that make HTTP calls
expecting success responses immediately after hot-reload:

* `HotReloadConfigRuntimePathsEndToEndTest`: REST and GraphQL calls now
retry up to 5 times with 1 second delays
* `HotReloadConfigConnectionString`: REST call now uses
`WaitForRestEndpointAsync` helper
* `HotReloadConfigDatabaseType`: REST call now uses
`WaitForRestEndpointAsync` helper

Added a shared `WaitForRestEndpointAsync` helper method that polls a
REST endpoint until it returns the expected status code (5 retries, 1
second delay).

This follows the same retry pattern already established in
`HotReloadConfigDataSource`, which had this exact fix applied
previously.


## How was this tested?

Ran a batch of 10 pipeline runs which all succeeded.

<img width="1597" height="790" alt="image"
src="https://github.com/user-attachments/assets/d7665b53-ba53-45b2-9c8c-333e4e296551"
/>

Author: aaronburtle

src/Service.Tests/Configuration/HotReload/ConfigurationHotReloadTests.cs

@@ -355,19 +355,20 @@ await WaitForConditionAsync(
         HttpResponseMessage badPathRestResult = await _testClient.GetAsync($"rest/Book");
         HttpResponseMessage badPathGQLResult = await _testClient.SendAsync(request);
 
-        HttpResponseMessage result = await _testClient.GetAsync($"{restPath}/Book");
+        // After hot-reload, the engine may still be re-initializing metadata providers.
+        // Poll the REST endpoint to allow time for the engine to become fully ready.
+        using HttpResponseMessage result = await WaitForRestEndpointAsync($"{restPath}/Book", HttpStatusCode.OK);
         string reloadRestContent = await result.Content.ReadAsStringAsync();
-        JsonElement reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync(
-            _testClient,
-            _configProvider,
-            GQL_QUERY_NAME,
-            GQL_QUERY);
+
+        // Poll the GraphQL endpoint to allow time for the engine to become fully ready.
+        (bool querySucceeded, JsonElement reloadGQLContents) = await WaitForGraphQLEndpointAsync(GQL_QUERY_NAME, GQL_QUERY);
 
         // Assert
         // Old paths are not found.
         Assert.AreEqual(HttpStatusCode.BadRequest, badPathRestResult.StatusCode);
         Assert.AreEqual(HttpStatusCode.NotFound, badPathGQLResult.StatusCode);
         // Hot reloaded paths return correct response.
+        Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload.");
         Assert.IsTrue(SqlTestHelper.JsonStringsDeepEqual(restBookContents, reloadRestContent));
         SqlTestHelper.PerformTestEqualJsonStrings(_bookDBOContents, reloadGQLContents.GetProperty("items").ToString());
     }
@@ -691,29 +692,8 @@ await WaitForConditionAsync(
         RuntimeConfig updatedRuntimeConfig = _configProvider.GetConfig();
         MsSqlOptions actualSessionContext = updatedRuntimeConfig.DataSource.GetTypedOptions<MsSqlOptions>();
 
-        // Retry GraphQL request because metadata re-initialization happens asynchronously
-        // after the "Validated hot-reloaded configuration file" message. The metadata provider
-        // factory clears and re-initializes providers on the hot-reload thread, so requests
-        // arriving before that completes will fail with "Initialization of metadata incomplete."
-        JsonElement reloadGQLContents = default;
-        bool querySucceeded = false;
-        for (int attempt = 1; attempt <= 10; attempt++)
-        {
-            reloadGQLContents = await GraphQLRequestExecutor.PostGraphQLRequestAsync(
-                _testClient,
-                _configProvider,
-                GQL_QUERY_NAME,
-                GQL_QUERY);
-
-            if (reloadGQLContents.ValueKind == JsonValueKind.Object &&
-                reloadGQLContents.TryGetProperty("items", out _))
-            {
-                querySucceeded = true;
-                break;
-            }
-
-            await Task.Delay(1000);
-        }
+        // Poll the GraphQL endpoint to allow time for the engine to become fully ready.
+        (bool querySucceeded, JsonElement reloadGQLContents) = await WaitForGraphQLEndpointAsync(GQL_QUERY_NAME, GQL_QUERY, maxRetries: 10);
 
         // Assert
         Assert.IsTrue(querySucceeded, "GraphQL query did not return valid results after hot-reload. Metadata initialization may not have completed.");
@@ -804,7 +784,9 @@ await WaitForConditionAsync(
             succeedConfigLog = _writer.ToString();
         }
 
-        HttpResponseMessage restResult = await _testClient.GetAsync("/rest/Book");
+        // After hot-reload, the engine may still be re-initializing metadata providers.
+        // Poll the REST endpoint to allow time for the engine to become fully ready.
+        using HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK);
 
         // Assert
         Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE));
@@ -897,7 +879,9 @@ await WaitForConditionAsync(
             succeedConfigLog = _writer.ToString();
         }
 
-        HttpResponseMessage restResult = await _testClient.GetAsync("/rest/Book");
+        // After hot-reload, the engine may still be re-initializing metadata providers.
+        // Poll the REST endpoint to allow time for the engine to become fully ready.
+        using HttpResponseMessage restResult = await WaitForRestEndpointAsync("/rest/Book", HttpStatusCode.OK);
 
         // Assert
         Assert.IsTrue(failedConfigLog.Contains(HOT_RELOAD_FAILURE_MESSAGE));
@@ -1033,4 +1017,84 @@ private static async Task WaitForConditionAsync(Func<bool> condition, TimeSpan t
 
         throw new TimeoutException("The condition was not met within the timeout period.");
     }
+
+    /// <summary>
+    /// Polls a REST endpoint until it returns the expected status code.
+    /// After a successful hot-reload, the engine may still be re-initializing
+    /// metadata providers, so an immediate request can intermittently fail.
+    /// </summary>
+    private static async Task<HttpResponseMessage> WaitForRestEndpointAsync(
+        string requestUri,
+        HttpStatusCode expectedStatus,
+        int maxRetries = 5,
+        int delayMilliseconds = 1000)
+    {
+        HttpResponseMessage response = null;
+        for (int attempt = 1; attempt <= maxRetries; attempt++)
+        {
+            response = await _testClient.GetAsync(requestUri);
+            if (response.StatusCode == expectedStatus)
+            {
+                return response;
+            }
+
+            Console.WriteLine($"REST {requestUri} returned {response.StatusCode} on attempt {attempt}/{maxRetries}, retrying...");
+
+            // Dispose unsuccessful responses to avoid leaking connections/sockets.
+            if (attempt < maxRetries)
+            {
+                response.Dispose();
+            }
+
+            await Task.Delay(delayMilliseconds);
+        }
+
+        // Return the last response (undisposed) so the caller can inspect/assert on it.
+        return response;
+    }
+
+    /// <summary>
+    /// Polls a GraphQL endpoint until it returns a valid response containing
+    /// the expected property. After a successful hot-reload, the engine may
+    /// still be re-initializing metadata providers, so an immediate request
+    /// can intermittently fail. PostGraphQLRequestAsync can also throw
+    /// (e.g. JsonException) if the server returns a non-JSON error response
+    /// during re-initialization.
+    /// </summary>
+    private static async Task<(bool Success, JsonElement Result)> WaitForGraphQLEndpointAsync(
+        string queryName,
+        string query,
+        string expectedProperty = "items",
+        int maxRetries = 5,
+        int delayMilliseconds = 1000)
+    {
+        JsonElement result = default;
+        for (int attempt = 1; attempt <= maxRetries; attempt++)
+        {
+            try
+            {
+                result = await GraphQLRequestExecutor.PostGraphQLRequestAsync(
+                    _testClient,
+                    _configProvider,
+                    queryName,
+                    query);
+
+                if (result.ValueKind == JsonValueKind.Object &&
+                    result.TryGetProperty(expectedProperty, out _))
+                {
+                    return (true, result);
+                }
+
+                Console.WriteLine($"GraphQL query returned {result.ValueKind} on attempt {attempt}/{maxRetries}, retrying...");
+            }
+            catch (Exception ex) when (ex is JsonException || ex is HttpRequestException)
+            {
+                Console.WriteLine($"GraphQL request threw {ex.GetType().Name} on attempt {attempt}/{maxRetries}: {ex.Message}");
+            }
+
+            await Task.Delay(delayMilliseconds);
+        }
+
+        return (false, result);
+    }
 }