Add validation and caching for log processing

- Added Microsoft.Extensions package references for caching and HTTP client support.
- Registered `SettingsService` with memory caching in `Program.cs`.
- Integrated `SettingsService` into `BatchingService` for log validation.
- Enhanced `ProcessNewBatchAsync` to include pre-validation of logs.
- Introduced `ValidateBatchAsync` for detailed log validation and error logging.
- Updated `BatchingServiceStats` to track total validation failures.
- Improved logging in `LogController` for validation errors and added endpoint for validation settings.
- Created `SettingsService` to fetch and cache validation settings from an API.
- Defined `ValidationSettings` for log validation constraints.
- Implemented `LogLineValidator` for validating log lines and batches.
- Added `ValidationResult` and `BatchValidationResult` classes for encapsulating validation results.

Author: Timothy Dodd

src/LogMkAgent/LogMkAgent.csproj

@@ -13,6 +13,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.8" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="9.0.8" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.8" />
     <PackageReference Include="System.Text.Json" Version="9.0.8" />

src/LogMkAgent/Program.cs

@@ -33,6 +33,8 @@ public static async Task Main(string[] args)
                 // Register services
                 services.AddSingleton<BatchingService>();
                 services.AddSingleton<HttpLogger>();
+                services.AddSingleton<SettingsService>();
+                services.AddMemoryCache(); // Required for SettingsService caching
 
                 // Configure HttpClient with typed client
                 services.AddHttpClient<LogApiClient>((serviceProvider, client) =>

src/LogMkAgent/Services/BatchService.cs

@@ -1,6 +1,7 @@
 using System.Collections.Concurrent;
 using LogMkCommon;
 using Microsoft.Extensions.Options;
+using LogMkAgent.Services;
 
 namespace LogMkAgent.Services;
 
@@ -10,6 +11,7 @@ public class BatchingService : IDisposable
     private readonly LogApiClient _httpClient;
     private readonly ILogger<BatchingService> _logger;
     private readonly BatchingOptions _options;
+    private readonly SettingsService _settingsService;
     private readonly Timer _timer;
     private readonly SemaphoreSlim _sendSemaphore = new(1, 1);
     private readonly CancellationTokenSource _cancellationTokenSource = new();
@@ -29,18 +31,21 @@ public class BatchingService : IDisposable
     private long _totalItemsProcessed;
     private long _totalBatchesSent;
     private long _totalFailures;
+    private long _totalValidationFailures;
     private DateTime _lastSuccessfulSend = DateTime.UtcNow;
 
     private volatile bool _disposed;
 
     public BatchingService(
         IOptions<BatchingOptions> batchingOptions,
         LogApiClient client,
-        ILogger<BatchingService> logger)
+        ILogger<BatchingService> logger,
+        SettingsService settingsService)
     {
         _httpClient = client ?? throw new ArgumentNullException(nameof(client));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _options = batchingOptions?.Value ?? throw new ArgumentNullException(nameof(batchingOptions));
+        _settingsService = settingsService ?? throw new ArgumentNullException(nameof(settingsService));
 
         ValidateOptions();
 
@@ -172,7 +177,21 @@ private async Task ProcessNewBatchAsync()
         if (currentBatch.Count == 0)
             return;
 
-        var success = await SendBatchWithRetryAsync(currentBatch, 1).ConfigureAwait(false);
+        // Pre-validate the batch before sending
+        var validatedBatch = await ValidateBatchAsync(currentBatch).ConfigureAwait(false);
+        if (validatedBatch.Count == 0)
+        {
+            _logger.LogWarning("Entire batch of {Count} logs failed validation, skipping send", currentBatch.Count);
+            return;
+        }
+
+        if (validatedBatch.Count < currentBatch.Count)
+        {
+            _logger.LogInformation("Pre-validation filtered {Filtered} invalid logs from batch of {Total}. Sending {Valid} valid logs.",
+                currentBatch.Count - validatedBatch.Count, currentBatch.Count, validatedBatch.Count);
+        }
+
+        var success = await SendBatchWithRetryAsync(validatedBatch, 1).ConfigureAwait(false);
 
         if (!success)
         {
@@ -203,6 +222,44 @@ private List<LogLine> ExtractBatch()
         return batch;
     }
 
+    private async Task<List<LogLine>> ValidateBatchAsync(List<LogLine> batch)
+    {
+        try
+        {
+            var validator = await _settingsService.GetValidatorAsync().ConfigureAwait(false);
+            if (validator == null)
+            {
+                _logger.LogWarning("Failed to get validator from settings service, sending batch without pre-validation");
+                return batch;
+            }
+
+            var validationResult = validator.ValidateBatch(batch);
+            
+            if (validationResult.InvalidCount > 0)
+            {
+                Interlocked.Add(ref _totalValidationFailures, validationResult.InvalidCount);
+                
+                // Log the first few validation errors for diagnostics
+                var errorSummary = validationResult.ValidationResults
+                    .SelectMany(r => r.Errors)
+                    .GroupBy(error => error)
+                    .ToDictionary(g => g.Key, g => g.Count());
+
+                _logger.LogWarning("Pre-validation failed for {InvalidCount}/{TotalCount} logs. " +
+                    "Error breakdown: {ErrorSummary}",
+                    validationResult.InvalidCount, validationResult.TotalCount,
+                    string.Join(", ", errorSummary.Select(kvp => $"{kvp.Key}: {kvp.Value}")));
+            }
+
+            return validationResult.ValidLogs;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error during batch validation, sending batch without pre-validation");
+            return batch;
+        }
+    }
+
     private async Task<bool> SendBatchWithRetryAsync(List<LogLine> batch, int attemptNumber)
     {
         if (batch.Count == 0)
@@ -306,6 +363,7 @@ public BatchingServiceStats GetStats()
                 TotalItemsProcessed = _totalItemsProcessed,
                 TotalBatchesSent = _totalBatchesSent,
                 TotalFailures = _totalFailures,
+                TotalValidationFailures = _totalValidationFailures,
                 PendingRetries = _retryQueue.Count,
                 LastSuccessfulSend = _lastSuccessfulSend
             };
@@ -363,6 +421,7 @@ public class BatchingServiceStats
     public long TotalItemsProcessed { get; set; }
     public long TotalBatchesSent { get; set; }
     public long TotalFailures { get; set; }
+    public long TotalValidationFailures { get; set; }
     public int PendingRetries { get; set; }
     public DateTime LastSuccessfulSend { get; set; }
 }

src/LogMkAgent/Services/SettingsService.cs

@@ -0,0 +1,66 @@
+using LogMkCommon;
+using Microsoft.Extensions.Caching.Memory;
+
+namespace LogMkAgent.Services;
+
+public class SettingsService
+{
+    private readonly LogApiClient _apiClient;
+    private readonly ILogger<SettingsService> _logger;
+    private readonly IMemoryCache _cache;
+    private readonly TimeSpan _cacheExpiry = TimeSpan.FromMinutes(30);
+    private const string SETTINGS_CACHE_KEY = "validation_settings";
+
+    public SettingsService(LogApiClient apiClient, ILogger<SettingsService> logger, IMemoryCache cache)
+    {
+        _apiClient = apiClient;
+        _logger = logger;
+        _cache = cache;
+    }
+
+    public async Task<ValidationSettings?> GetValidationSettingsAsync(CancellationToken cancellationToken = default)
+    {
+        // Try to get from cache first
+        if (_cache.TryGetValue(SETTINGS_CACHE_KEY, out ValidationSettings? cachedSettings))
+        {
+            _logger.LogDebug("Using cached validation settings");
+            return cachedSettings;
+        }
+
+        try
+        {
+            _logger.LogDebug("Fetching validation settings from API");
+            var settings = await _apiClient.GetDataAsync<ValidationSettings>("api/log/settings");
+            
+            // Cache the settings
+            _cache.Set(SETTINGS_CACHE_KEY, settings, _cacheExpiry);
+            
+            _logger.LogInformation("Fetched validation settings: MaxDaysOld={MaxDaysOld}, MaxBatchSize={MaxBatchSize}, Version={Version}",
+                settings.MaxDaysOld, settings.MaxBatchSize, settings.Version);
+            
+            return settings;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex, "Failed to fetch validation settings from API, using defaults");
+            
+            // Return default settings if API call fails
+            var defaultSettings = new ValidationSettings();
+            _cache.Set(SETTINGS_CACHE_KEY, defaultSettings, TimeSpan.FromMinutes(5)); // Shorter cache for fallback
+            
+            return defaultSettings;
+        }
+    }
+
+    public void InvalidateCache()
+    {
+        _cache.Remove(SETTINGS_CACHE_KEY);
+        _logger.LogDebug("Validation settings cache invalidated");
+    }
+
+    public async Task<LogLineValidator?> GetValidatorAsync(CancellationToken cancellationToken = default)
+    {
+        var settings = await GetValidationSettingsAsync(cancellationToken);
+        return settings != null ? new LogLineValidator(settings) : null;
+    }
+}

src/LogMkApi/Controllers/LogController.cs

@@ -179,8 +179,17 @@ public async Task<ActionResult<LogResponse>> Create(
         // Log validation errors for monitoring
         if (errors.Any())
         {
-            _logger.LogWarning("Batch {BatchId}: {ErrorCount} validation errors, {SkippedCount} logs skipped",
-                batchId, errors.Count, skippedCount);
+            // Group errors by type for better insights
+            var errorSummary = errors
+                .SelectMany(e => e.Errors)
+                .GroupBy(error => error)
+                .ToDictionary(g => g.Key, g => g.Count());
+
+            _logger.LogWarning("Batch {BatchId}: {ErrorCount} validation errors, {SkippedCount} logs skipped. " +
+                "Error breakdown: {ErrorSummary}", 
+                batchId, errors.Count, skippedCount, 
+                string.Join(", ", errorSummary.Select(kvp => $"{kvp.Key}: {kvp.Value}")));
+            
             _metrics.IncrementErrors("validation", skippedCount);
         }
 
@@ -386,6 +395,29 @@ public async Task<IEnumerable<LatestDeploymentEntry>> GetLatestEntryTimes()
         var entries = await _logRepo.GetLatestEntryTimes();
         return entries;
     }
+
+    [AllowAnonymous]
+    [HttpGet("settings")]
+    public IActionResult GetValidationSettings()
+    {
+        var settings = new LogMkCommon.ValidationSettings
+        {
+            MaxDaysOld = LogMaxDaysOld,
+            MaxFutureMinutes = 5,
+            MaxLineLength = 10000,
+            MaxDeploymentNameLength = 100,
+            MaxPodNameLength = 100,
+            DeploymentNamePattern = @"^[a-zA-Z0-9\-._]+$",
+            PodNamePattern = @"^[a-zA-Z0-9\-._]+$",
+            AllowEmptyLogLevel = false,
+            MaxBatchSize = 1000,
+            Version = "1.0",
+            LastUpdated = DateTime.UtcNow
+        };
+
+        _logger.LogDebug("Validation settings requested by agent");
+        return Ok(settings);
+    }
     [HttpGet("pods")]
     public async Task<IEnumerable<Pod>> GetPods()
     {