harden responses

Author: elasticroentgen

PoolManager.cs

@@ -93,6 +93,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             {
                 _logger.LogInformation("Cleaning runners...");
 
+                var staleRemoved = _queues.CreatedRunners.RemoveStale(TimeSpan.FromMinutes(15));
+                if (staleRemoved > 0)
+                    _logger.LogWarning("Removed {Count} stale entries from CreatedRunnersTracking", staleRemoved);
+
                 var checkInId = SentrySdk.CaptureCheckIn("CheckForStuckRunners", CheckInStatus.InProgress);
                 try
                 {
@@ -106,13 +110,25 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                     SentrySdk.CaptureException(ex);
                 }
 
-                await CleanUpRunners(targetConfig);
-                
-                await StartPoolRunners(targetConfig);
+                try
+                {
+                    await CleanUpRunners(targetConfig);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError($"Unable to clean up runners: {ex.GetFullExceptionDetails()}");
+                    SentrySdk.CaptureException(ex);
+                }
 
-                var staleRemoved = _queues.CreatedRunners.RemoveStale(TimeSpan.FromMinutes(30));
-                if (staleRemoved > 0)
-                    _logger.LogWarning("Removed {Count} stale entries from CreatedRunnersTracking", staleRemoved);
+                try
+                {
+                    await StartPoolRunners(targetConfig);
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError($"Unable to start pool runners: {ex.GetFullExceptionDetails()}");
+                    SentrySdk.CaptureException(ex);
+                }
 
                 await CheckForStuckJobs(targetConfig);
 

Program.cs

@@ -459,6 +459,12 @@ private static async Task<IResult> RunnerStateReportHandler(HttpRequest request,
     {
         await using var db = new ActionsRunnerContext();
         var runner = await db.Runners.Include(x => x.Lifecycle).Include(x => x.Job).FirstOrDefaultAsync(x => x.Hostname == hostname);
+        if (runner == null)
+        {
+            logger.LogError("Runner {Hostname} not found in database during state report ({State}). Removing tracking entry.", hostname, state);
+            runnerQueue.CreatedRunners.Remove(hostname, out _);
+            return Results.StatusCode(404);
+        }
         switch (state)
         {
             case "ok":
@@ -560,6 +566,10 @@ private static async Task<IResult> RunnerStateReportHandler(HttpRequest request,
                     await db.SaveChangesAsync();
                 }
 
+                break;
+            default:
+                logger.LogWarning("Runner {Hostname} reported unknown state '{State}'. Removing tracking entry.", hostname, state);
+                runnerQueue.CreatedRunners.Remove(hostname, out _);
                 break;
         }