Skip to content

Commit e004fec

Browse files
committed
Fix HttpRetryPolicy: fail fast on unreachable services
- Add per-attempt timeout support and Ollama-specific timeout - Avoid retrying connection-refused/host-not-found to prevent long-running test hangs
1 parent d341f7f commit e004fec

3 files changed

Lines changed: 37 additions & 4 deletions

File tree

src/Core/Constants.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,18 @@ public static class HttpRetryDefaults
220220
/// </summary>
221221
public const int MaxAttempts = 5;
222222

223+
/// <summary>
224+
/// Default timeout per attempt (seconds).
225+
/// Applied by <see cref="KernelMemory.Core.Http.HttpRetryPolicy"/> to avoid hanging calls.
226+
/// </summary>
227+
public const int DefaultPerAttemptTimeoutSeconds = 60;
228+
229+
/// <summary>
230+
/// Per-attempt timeout for local Ollama calls (seconds).
231+
/// Keep this low so local development and tests fail fast when Ollama is not running.
232+
/// </summary>
233+
public const int OllamaPerAttemptTimeoutSeconds = 5;
234+
223235
/// <summary>
224236
/// Base delay for exponential backoff.
225237
/// </summary>

src/Core/Embeddings/Providers/OllamaEmbeddingGenerator.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,8 @@ public async Task<EmbeddingResult> GenerateAsync(string text, CancellationToken
9292
},
9393
this._logger,
9494
ct,
95-
delayAsync: this._delayAsync).ConfigureAwait(false);
95+
delayAsync: this._delayAsync,
96+
perAttemptTimeout: TimeSpan.FromSeconds(Constants.HttpRetryDefaults.OllamaPerAttemptTimeoutSeconds)).ConfigureAwait(false);
9697

9798
response.EnsureSuccessStatusCode();
9899

src/Core/Http/HttpRetryPolicy.cs

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

33
using System.Net;
4+
using System.Net.Sockets;
45
using Microsoft.Extensions.Logging;
56

67
namespace KernelMemory.Core.Http;
@@ -19,13 +20,15 @@ public static async Task<HttpResponseMessage> SendAsync(
1920
Func<HttpRequestMessage> requestFactory,
2021
ILogger logger,
2122
CancellationToken ct,
22-
Func<TimeSpan, CancellationToken, Task>? delayAsync = null)
23+
Func<TimeSpan, CancellationToken, Task>? delayAsync = null,
24+
TimeSpan? perAttemptTimeout = null)
2325
{
2426
ArgumentNullException.ThrowIfNull(httpClient, nameof(httpClient));
2527
ArgumentNullException.ThrowIfNull(requestFactory, nameof(requestFactory));
2628
ArgumentNullException.ThrowIfNull(logger, nameof(logger));
2729

2830
delayAsync ??= Task.Delay;
31+
perAttemptTimeout ??= TimeSpan.FromSeconds(Constants.HttpRetryDefaults.DefaultPerAttemptTimeoutSeconds);
2932

3033
Exception? lastException = null;
3134

@@ -37,7 +40,10 @@ public static async Task<HttpResponseMessage> SendAsync(
3740

3841
try
3942
{
40-
var response = await httpClient.SendAsync(request, ct).ConfigureAwait(false);
43+
using var attemptCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
44+
attemptCts.CancelAfter(perAttemptTimeout.Value);
45+
46+
var response = await httpClient.SendAsync(request, attemptCts.Token).ConfigureAwait(false);
4147

4248
if (response.IsSuccessStatusCode)
4349
{
@@ -81,7 +87,7 @@ public static async Task<HttpResponseMessage> SendAsync(
8187
catch (HttpRequestException ex)
8288
{
8389
lastException = ex;
84-
if (attempt == Constants.HttpRetryDefaults.MaxAttempts)
90+
if (!IsRetryableException(ex) || attempt == Constants.HttpRetryDefaults.MaxAttempts)
8591
{
8692
throw;
8793
}
@@ -101,6 +107,20 @@ public static async Task<HttpResponseMessage> SendAsync(
101107
throw lastException ?? new HttpRequestException("HTTP call failed after retries");
102108
}
103109

110+
private static bool IsRetryableException(HttpRequestException ex)
111+
{
112+
// Fail fast on common "service not running" / "unreachable" conditions.
113+
// Retrying these (especially with default HttpClient timeouts) can make test runs appear hung.
114+
if (ex.InnerException is SocketException socketEx)
115+
{
116+
return socketEx.SocketErrorCode != SocketError.ConnectionRefused &&
117+
socketEx.SocketErrorCode != SocketError.HostNotFound &&
118+
socketEx.SocketErrorCode != SocketError.NetworkUnreachable;
119+
}
120+
121+
return true;
122+
}
123+
104124
private static bool IsRetryableStatusCode(HttpStatusCode statusCode)
105125
{
106126
return statusCode == HttpStatusCode.TooManyRequests ||

0 commit comments

Comments
 (0)