add a created time to fix stuck runners

Author: elasticroentgen

Database/Runner.cs

@@ -16,13 +16,20 @@ public class Runner
     public ICollection<RunnerLifecycle> Lifecycle { get; set; }
     public long CloudServerId { get; set; }
 
-    public DateTime CreateTime
+    public DateTime CreationQueuedTime
     {
         get
         {
             return Lifecycle.FirstOrDefault(x => x.Status == RunnerStatus.CreationQueued)!.EventTimeUtc;
         }
     }
+    public DateTime CreatedTime
+    {
+        get
+        {
+            return Lifecycle.FirstOrDefault(x => x.Status == RunnerStatus.Created)!.EventTimeUtc;
+        }
+    }
 
     public RunnerStatus LastState
     {

PoolManager.cs

@@ -151,7 +151,7 @@ private async Task CheckForStuckRunners(List<GithubTargetConfiguration> targetCo
         {
 
             // check if runner is old enough to be stuck
-            if (stuckRunner.CreateTime + TimeSpan.FromMinutes(5) > DateTime.UtcNow)
+            if (stuckRunner.CreatedTime + TimeSpan.FromMinutes(10) > DateTime.UtcNow)
                 continue;
 
             // Note stuckness in lifecycle and add runner to deletion queue
@@ -453,7 +453,7 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
                     continue;
                 }
 
-                if (DateTime.UtcNow - runner.CreateTime < TimeSpan.FromMinutes(30))
+                if (DateTime.UtcNow - runner.CreationQueuedTime < TimeSpan.FromMinutes(30))
                 {
                     // VM younger than 30min - not cleaning yet
                     continue;
@@ -488,7 +488,7 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
                     continue;
                 }
 
-                if (DateTime.UtcNow - runner.CreateTime < TimeSpan.FromHours(6))
+                if (DateTime.UtcNow - runner.CreationQueuedTime < TimeSpan.FromHours(6))
                 {
                     // VM younger than 6h - not cleaning yet
                     continue;
@@ -622,7 +622,7 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
 
         foreach (var onlineSrvFromDb in db.Runners.Include(x => x.Lifecycle).Where(x => x.IsOnline))
         {
-            if(onlineSrvFromDb.CreateTime + TimeSpan.FromHours(1) > DateTime.UtcNow ) continue; // Leave young runners alone
+            if(onlineSrvFromDb.CreationQueuedTime + TimeSpan.FromHours(1) > DateTime.UtcNow ) continue; // Leave young runners alone
             if (registeredServerNames.Contains(onlineSrvFromDb.Hostname)) continue;
 
             if(onlineSrvFromDb.LastState == RunnerStatus.DeletionQueued) continue;

Program.cs

@@ -535,7 +535,7 @@ await db.Jobs.AddAsync(new Job
 
         ProcessedJobCount.Labels(job.Owner, jobRunner.Size).Inc();
 
-        double secondsAlive = (DateTime.UtcNow - jobRunner.CreateTime).TotalSeconds;
+        double secondsAlive = (DateTime.UtcNow - jobRunner.CreationQueuedTime).TotalSeconds;
         TotalMachineTime.Labels(job.Owner, jobRunner.Size).Inc(secondsAlive);
 
     }