improved stuck job handling

Author: elasticroentgen

Models/RunnerQueue.cs

@@ -68,6 +68,17 @@ public void Enqueue(CreateRunnerTask task)
         context.SaveChanges();
     }
 
+    public int CountMatchingRunners(string size, string owner, string profile)
+    {
+        using var context = new ActionsRunnerContext();
+        return context.CreateTaskQueues
+            .Join(context.Runners,
+                ctq => ctq.RunnerDbId,
+                r => r.RunnerId,
+                (ctq, r) => r)
+            .Count(r => r.Size == size && r.Owner == owner && r.Profile == profile);
+    }
+
     public bool TryDequeue(out CreateRunnerTask? task)
     {
         using var context = new ActionsRunnerContext();
@@ -229,6 +240,17 @@ public bool Remove(string hostname, out CreateRunnerTask? task)
     {
         return TryRemove(hostname, out task);
     }
+
+    public int CountMatchingRunners(string size, string owner, string profile)
+    {
+        using var context = new ActionsRunnerContext();
+        return context.CreatedRunnersTrackings
+            .Join(context.Runners,
+                crt => crt.RunnerDbId,
+                r => r.RunnerId,
+                (crt, r) => r)
+            .Count(r => r.Size == size && r.Owner == owner && r.Profile == profile);
+    }
 }
 
 // Wrapper for CancelledRunnersCounter to mimic ConcurrentDictionary<(string, string, string, string, string), int>

PoolManager.cs

@@ -593,6 +593,26 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
                 continue;
             }
 
+            // Count-based check: compare matching runners in pipeline vs stuck jobs needing them
+            var profile = stuckJob.RequestedProfile ?? "default";
+            int matchingRunnersInPipeline =
+                _queues.CreateTasks.CountMatchingRunners(stuckJob.RequestedSize, stuckJob.Owner, profile)
+                + _queues.CreatedRunners.CountMatchingRunners(stuckJob.RequestedSize, stuckJob.Owner, profile);
+
+            int stuckJobsWithSameRequirements = stuckJobs.Count(j =>
+                j.RequestedSize == stuckJob.RequestedSize
+                && j.Owner == stuckJob.Owner
+                && (j.RequestedProfile ?? "default") == profile);
+
+            if (matchingRunnersInPipeline >= stuckJobsWithSameRequirements)
+            {
+                _logger.LogInformation(
+                    "Enough runners in pipeline ({RunnerCount}) for stuck jobs ({JobCount}) with Size={Size}, Owner={Owner}, Profile={Profile}. Skipping replacement for job {JobId}.",
+                    matchingRunnersInPipeline, stuckJobsWithSameRequirements,
+                    stuckJob.RequestedSize, stuckJob.Owner, profile, stuckJob.JobId);
+                continue;
+            }
+
             int replacementsInQueue =  _queues.CreateTasks.CountWhere(x => x.IsStuckReplacement);
             if (replacementsInQueue > 25)
             {
@@ -633,7 +653,6 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
                 continue;
             }
 
-            var profile = stuckJob.RequestedProfile ?? "default";
             string arch = Program.Config.Sizes.FirstOrDefault(x => x.Name == stuckJob.RequestedSize)?.Arch;
             Runner newRunner = new()
             {