RE1-T118 TTS worker fix

Author: ucswift

Core/Resgrid.Services/TtsAudioService.cs

@@ -74,6 +74,9 @@ public async Task RegenerateStaticPromptsAsync(IEnumerable<string> prompts, Canc
 
 		private static Exception CreateRequestFailure(string operation, RestResponse response)
 		{
+			if (response.ErrorException is OperationCanceledException or TaskCanceledException)
+				return new OperationCanceledException($"The TTS service {operation} was canceled.", response.ErrorException);
+
 			var status = response.StatusCode == 0 ? "no-response" : response.StatusCode.ToString();
 			var detail = response.ErrorException?.Message;
 

Tests/Resgrid.Tests/Workers/Console/Tasks/TtsStaticPromptRefreshTaskTests.cs

@@ -27,16 +27,19 @@ public class TtsStaticPromptRefreshTaskTests
 		private IContainer _testWorkerContainer;
 		private string _originalServiceBaseUrl;
 		private string _originalStaticPromptAdminKey;
+		private TimeSpan _originalRetryDelay;
 
 		[SetUp]
 		public void SetUp()
 		{
 			_originalWorkerContainer = WorkerBootstrapperContainerField.GetValue(null) as IContainer;
 			_originalServiceBaseUrl = TtsConfig.ServiceBaseUrl;
 			_originalStaticPromptAdminKey = TtsConfig.StaticPromptAdminKey;
+			_originalRetryDelay = TtsStaticPromptRefreshTask.RetryDelay;
 
 			TtsConfig.ServiceBaseUrl = "https://tts.example.com";
 			TtsConfig.StaticPromptAdminKey = "prompt-admin-key";
+			TtsStaticPromptRefreshTask.RetryDelay = TimeSpan.FromMilliseconds(1);
 		}
 
 		[TearDown]
@@ -46,10 +49,11 @@ public void TearDown()
 			_testWorkerContainer?.Dispose();
 			TtsConfig.ServiceBaseUrl = _originalServiceBaseUrl;
 			TtsConfig.StaticPromptAdminKey = _originalStaticPromptAdminKey;
+			TtsStaticPromptRefreshTask.RetryDelay = _originalRetryDelay;
 		}
 
 		[Test]
-		public async Task process_async_should_rethrow_refresh_failures()
+		public async Task process_async_should_throw_after_all_retries_exhausted()
 		{
 			var failure = new InvalidOperationException("refresh failed");
 			var ttsAudioService = new Mock<ITtsAudioService>(MockBehavior.Strict);
@@ -66,6 +70,10 @@ await FluentActions
 				.Should()
 				.ThrowAsync<InvalidOperationException>()
 				.WithMessage("refresh failed");
+
+			ttsAudioService.Verify(
+				x => x.RegenerateStaticPromptsAsync(It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()),
+				Times.Exactly(3));
 		}
 
 		[Test]
@@ -85,6 +93,40 @@ await FluentActions
 				.Awaiting(() => task.ProcessAsync(new TtsStaticPromptRefreshCommand(1), progress.Object, cancellationTokenSource.Token))
 				.Should()
 				.ThrowAsync<OperationCanceledException>();
+
+			ttsAudioService.Verify(
+				x => x.RegenerateStaticPromptsAsync(It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()),
+				Times.Once);
+		}
+
+		[Test]
+		public async Task process_async_should_succeed_on_retry_after_initial_failure()
+		{
+			var ttsAudioService = new Mock<ITtsAudioService>(MockBehavior.Strict);
+			var callCount = 0;
+			ttsAudioService
+				.Setup(x => x.RegenerateStaticPromptsAsync(It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
+				.Returns(() =>
+				{
+					callCount++;
+					if (callCount == 1)
+						throw new InvalidOperationException("transient failure");
+
+					return Task.CompletedTask;
+				});
+			SetWorkerContainer(ttsAudioService.Object);
+
+			var task = new TtsStaticPromptRefreshTask(Mock.Of<ILogger>());
+			var progress = new Mock<IQuidjiboProgress>(MockBehavior.Loose);
+
+			await FluentActions
+				.Awaiting(() => task.ProcessAsync(new TtsStaticPromptRefreshCommand(1), progress.Object, CancellationToken.None))
+				.Should()
+				.NotThrowAsync();
+
+			ttsAudioService.Verify(
+				x => x.RegenerateStaticPromptsAsync(It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()),
+				Times.Exactly(2));
 		}
 
 		private void SetWorkerContainer(ITtsAudioService ttsAudioService)

Workers/Resgrid.Workers.Console/Tasks/TtsStaticPromptRefreshTask.cs

@@ -15,6 +15,9 @@ namespace Resgrid.Workers.Console.Tasks
 {
 	public class TtsStaticPromptRefreshTask : IQuidjiboHandler<TtsStaticPromptRefreshCommand>
 	{
+		public const int MaxRetries = 3;
+		public static TimeSpan RetryDelay = TimeSpan.FromSeconds(30);
+
 		public string Name => "TTS Static Prompt Refresh";
 		public int Priority => 1;
 		private readonly ILogger _logger;
@@ -38,11 +41,41 @@ public async Task ProcessAsync(TtsStaticPromptRefreshCommand command, IQuidjiboP
 				}
 
 				var ttsAudioService = Bootstrapper.GetKernel().Resolve<ITtsAudioService>();
+				var prompts = TwilioVoicePromptCatalog.GetStaticPrompts();
+				Exception lastException = null;
+
+				for (int attempt = 1; attempt <= MaxRetries; attempt++)
+				{
+					cancellationToken.ThrowIfCancellationRequested();
 
-				_logger.LogInformation("TtsStaticPromptRefresh::Refreshing static prompts");
-				await ttsAudioService.RegenerateStaticPromptsAsync(TwilioVoicePromptCatalog.GetStaticPrompts(), cancellationToken);
+					try
+					{
+						_logger.LogInformation("TtsStaticPromptRefresh::Refreshing static prompts (attempt {Attempt}/{MaxRetries})", attempt, MaxRetries);
+						await ttsAudioService.RegenerateStaticPromptsAsync(prompts, cancellationToken);
+
+						_logger.LogInformation("TtsStaticPromptRefresh::Successfully refreshed static prompts on attempt {Attempt}", attempt);
+						progress.Report(100, $"Finishing the {Name} Task");
+						return;
+					}
+					catch (OperationCanceledException)
+					{
+						throw;
+					}
+					catch (Exception ex)
+					{
+						lastException = ex;
+
+						if (attempt < MaxRetries)
+						{
+							_logger.LogWarning(ex, "TtsStaticPromptRefresh::Attempt {Attempt}/{MaxRetries} failed, retrying in {DelaySeconds}s", attempt, MaxRetries, (int)RetryDelay.TotalSeconds);
+							await Task.Delay(RetryDelay, cancellationToken);
+						}
+					}
+				}
 
-				progress.Report(100, $"Finishing the {Name} Task");
+				Resgrid.Framework.Logging.LogException(lastException);
+				_logger.LogError(lastException, "TtsStaticPromptRefresh::Failed to refresh static prompts after {MaxRetries} attempts", MaxRetries);
+				throw lastException!;
 			}
 			catch (OperationCanceledException)
 			{