Harden Azure OpenAI embedding retry for 429 throttling (#1114)

## Why
Embedding generation could fail fast on transient Azure OpenAI
throttling (HTTP 429), which interrupted full vector rebuild runs. We
need resilient retry behavior that respects service guidance while still
failing clearly when retries are exhausted.

## What changed
- Added robust retry handling in `EmbeddingService` for transient
failures, including `ClientResultException` status-based detection
(429/5xx/408), exponential backoff, jitter, and Retry-After header
support when present.
- Kept failure behavior explicit: retries are logged with context, and
final exhaustion throws a clear terminal exception with attempt details.
- Preserved cancellation semantics: caller-requested cancellation is
rethrown directly and not wrapped as retry exhaustion.
- Improved staging cleanup behavior after failures so cleanup is best
effort and does not mask the original error.
- Replaced ad-hoc logger calls with source-generated `LoggerMessage`
methods to match project logging conventions.

## Configuration and ASP.NET conventions
- Introduced `EmbeddingRetryOptions` and bound it via standard options
binding at `AIOptions:EmbeddingRetry`.
- Added validation (data annotations plus runtime validation) and safe
defaults.
- Added `EmbeddingRetry` defaults in
`EssentialCSharp.Web/appsettings.json`.
- Added `MaxDelayMs` cap to prevent delay overflow/unbounded waits.

## Additional cleanup
- Removed tracked `build_output.txt` and added it to `.gitignore`.
- Removed obsolete `RetryOptions` model after renaming to avoid
ambiguity with `Azure.Core.RetryOptions`.

## Validation
- Built `EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj`
successfully after changes.
- Ran dual review passes with Opus 4.6 and GPT-5.5 and incorporated
findings.

Author: BenjaminMichaelis

.gitignore

@@ -32,6 +32,7 @@ TestResults/
 !Michaelis_TableOfContents.docx
 
 # Old or generated files to not commit
+build_output.txt
 wwwroot/sitemap.xml
 wwwroot/Chapters
 EssentialCSharp.Web/wwwroot/Chapters

EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs

@@ -1,10 +1,12 @@
 using Azure.AI.OpenAI;
 using Azure.Core;
 using Azure.Identity;
+using EssentialCSharp.Chat.Common.Models;
 using EssentialCSharp.Chat.Common.Services;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
 using Microsoft.SemanticKernel;
 using Npgsql;
 
@@ -65,6 +67,15 @@ public static IServiceCollection AddAzureOpenAIServices(
             .UseOpenTelemetry();
 #pragma warning restore SKEXP0010
 
+        // Ensure options are available even when caller provides AIOptions directly.
+        services.AddOptions<EmbeddingRetryOptions>()
+            .ValidateDataAnnotations()
+            .Validate(options =>
+            {
+                options.Validate();
+                return true;
+            }, "Embedding retry configuration is invalid.");
+
         // Register shared AI services
         services.AddSingleton<EmbeddingService>();
         services.AddSingleton<AISearchService>();
@@ -89,6 +100,17 @@ public static IServiceCollection AddAzureOpenAIServices(
         // Configure AI options from configuration
         services.Configure<AIOptions>(configuration.GetSection("AIOptions"));
 
+        // Configure retry options from configuration section
+        // Environment variables like EmbeddingRetry:MaxRetries will override defaults
+        services.AddOptions<EmbeddingRetryOptions>()
+            .Bind(configuration.GetSection(EmbeddingRetryOptions.SectionPath))
+            .ValidateDataAnnotations()
+            .Validate(options =>
+            {
+                options.Validate();
+                return true;
+            }, "Embedding retry configuration is invalid.");
+
         var aiOptions = configuration.GetSection("AIOptions").Get<AIOptions>();
         if (aiOptions == null)
         {

EssentialCSharp.Chat.Shared/Models/EmbeddingRetryOptions.cs

@@ -0,0 +1,83 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace EssentialCSharp.Chat.Common.Models;
+
+/// <summary>
+/// Configuration options for retry logic when calling external services like Azure OpenAI.
+/// </summary>
+public sealed class EmbeddingRetryOptions
+{
+    /// <summary>
+    /// Configuration section path in appsettings.json.
+    /// </summary>
+    public const string SectionPath = "AIOptions:EmbeddingRetry";
+
+    /// <summary>
+    /// Maximum number of retries for transient failures.
+    /// Default is 5 retries (initial attempt + 5 retries = 6 total attempts).
+    /// </summary>
+    [Range(0, 20)]
+    public int MaxRetries { get; set; } = 5;
+
+    /// <summary>
+    /// Base delay in milliseconds before the first retry.
+    /// Subsequent retries use exponential backoff: baseDelay * (backoffMultiplier ^ attemptNumber).
+    /// Default is 1000ms (1 second).
+    /// </summary>
+    [Range(0, 600000)]
+    public int BaseDelayMs { get; set; } = 1000;
+
+    /// <summary>
+    /// Maximum delay in milliseconds for exponential backoff before jitter.
+    /// This caps retry delays to avoid overflow and unbounded waits.
+    /// </summary>
+    [Range(1, 600000)]
+    public int MaxDelayMs { get; set; } = 60000;
+
+    /// <summary>
+    /// Exponential backoff multiplier. Each retry delay is multiplied by this value.
+    /// For example, with baseDelay=1000ms and multiplier=2.0:
+    /// - 1st retry: 1000ms
+    /// - 2nd retry: 2000ms
+    /// - 3rd retry: 4000ms
+    /// - 4th retry: 8000ms
+    /// Default is 2.0 (double each time).
+    /// </summary>
+    [Range(1.0, 10.0)]
+    public double BackoffMultiplier { get; set; } = 2.0;
+
+    /// <summary>
+    /// Maximum jitter fraction added to each retry delay to prevent thundering herd.
+    /// Jitter is a random value in range [0, maxDelay * maxJitterFraction].
+    /// For example, with maxJitterFraction=0.2 and delay=1000ms:
+    /// actual delay will be between 1000ms and 1200ms.
+    /// Default is 0.2 (20% jitter).
+    /// </summary>
+    [Range(0.0, 1.0)]
+    public double MaxJitterFraction { get; set; } = 0.2;
+
+    /// <summary>
+    /// Validates that configuration values are reasonable.
+    /// </summary>
+    /// <exception cref="InvalidOperationException">Thrown if configuration is invalid.</exception>
+    public void Validate()
+    {
+        if (MaxRetries < 0)
+            throw new InvalidOperationException("MaxRetries must be non-negative.");
+
+        if (BaseDelayMs < 0)
+            throw new InvalidOperationException("BaseDelayMs must be non-negative.");
+
+        if (MaxDelayMs <= 0)
+            throw new InvalidOperationException("MaxDelayMs must be positive.");
+
+        if (BaseDelayMs > MaxDelayMs)
+            throw new InvalidOperationException("BaseDelayMs must be less than or equal to MaxDelayMs.");
+
+        if (BackoffMultiplier < 1.0)
+            throw new InvalidOperationException("BackoffMultiplier must be >= 1.0.");
+
+        if (MaxJitterFraction < 0.0 || MaxJitterFraction > 1.0)
+            throw new InvalidOperationException("MaxJitterFraction must be between 0.0 and 1.0.");
+    }
+}