add job vanish detection

Author: elasticroentgen

GitHubApi.cs

@@ -135,7 +135,7 @@ public static async Task<bool> RemoveRunnerFromRepo(string repoName, string orgG
         return response.IsSuccessStatusCode;
     }
 
-    public static async Task<GitHubApiWorkflowRun> GetJobInfoForOrg(long stuckJobGithubJobId,string repoName, string orgGitHubToken)
+    public static async Task<(GitHubApiWorkflowRun Job, System.Net.HttpStatusCode StatusCode)> GetJobInfoForOrg(long stuckJobGithubJobId,string repoName, string orgGitHubToken)
     {
         using var request = CreateRequest(HttpMethod.Get, $"https://api.github.com/orgs/{repoName}/actions/jobs/{stuckJobGithubJobId}", orgGitHubToken);
         HttpResponseMessage response = await SharedClient.SendAsync(request);
@@ -144,13 +144,13 @@ public static async Task<GitHubApiWorkflowRun> GetJobInfoForOrg(long stuckJobGit
             string content = await response.Content.ReadAsStringAsync();
             GitHubApiWorkflowRun responseObject = JsonSerializer.Deserialize<GitHubApiWorkflowRun>(content);
 
-            return responseObject;
+            return (responseObject, response.StatusCode);
         }
         Log.Warning($"Unable to get GH job info for {repoName}/{stuckJobGithubJobId}: [{response.StatusCode}] {response.ReasonPhrase}");
 
-        return null;
+        return (null, response.StatusCode);
     }
-    public static async Task<GitHubApiWorkflowRun> GetJobInfoForRepo(long stuckJobGithubJobId,string repoName, string orgGitHubToken)
+    public static async Task<(GitHubApiWorkflowRun Job, System.Net.HttpStatusCode StatusCode)> GetJobInfoForRepo(long stuckJobGithubJobId,string repoName, string orgGitHubToken)
     {
         using var request = CreateRequest(HttpMethod.Get, $"https://api.github.com/repos/{repoName}/actions/jobs/{stuckJobGithubJobId}", orgGitHubToken);
         HttpResponseMessage response = await SharedClient.SendAsync(request);
@@ -159,10 +159,10 @@ public static async Task<GitHubApiWorkflowRun> GetJobInfoForRepo(long stuckJobGi
             string content = await response.Content.ReadAsStringAsync();
             GitHubApiWorkflowRun responseObject = JsonSerializer.Deserialize<GitHubApiWorkflowRun>(content);
 
-            return responseObject;
+            return (responseObject, response.StatusCode);
         }
         Log.Warning($"Unable to get GH job info for {repoName}/{stuckJobGithubJobId}: [{response.StatusCode}] {response.ReasonPhrase}");
 
-        return null;
+        return (null, response.StatusCode);
     }
 }

PoolManager.cs

@@ -607,12 +607,19 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
             }
 
             // Verify job is still queued on GitHub before treating it as stuck
-            GitHubApiWorkflowRun ghJob = await GitHubApi.GetJobInfoForRepo(stuckJob.GithubJobId, stuckJob.Repository , owner.GitHubToken);
+            var (ghJob, ghStatusCode) = await GitHubApi.GetJobInfoForRepo(stuckJob.GithubJobId, stuckJob.Repository , owner.GitHubToken);
             if (ghJob == null || ghJob.Status != "queued")
             {
-                if (ghJob == null)
+                if (ghJob == null && ghStatusCode == System.Net.HttpStatusCode.NotFound)
                 {
-                    _logger.LogWarning($"GHjob for {stuckJob.JobId} is null - not actually stuck");
+                    _logger.LogWarning($"Job {stuckJob.GithubJobId} not found on GitHub (404) - marking as vanished.");
+                    stuckJob.State = JobState.Vanished;
+                    stuckJob.CompleteTime = DateTime.UtcNow;
+                    await db.SaveChangesAsync();
+                }
+                else if (ghJob == null)
+                {
+                    _logger.LogWarning($"Unable to verify job {stuckJob.JobId} on GitHub (HTTP {(int)ghStatusCode}) - skipping this cycle");
                 }
                 else if (ghJob.Status == "completed")
                 {
@@ -1137,7 +1144,7 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
                             var targetConfig = Program.Config.TargetConfigs.FirstOrDefault(x => x.Name == runner.Job.Owner);
                             if (targetConfig != null)
                             {
-                                var githubJob = await GitHubApi.GetJobInfoForOrg(runner.Job.GithubJobId, runner.Job.Repository, targetConfig.GitHubToken);
+                                var (githubJob, _) = await GitHubApi.GetJobInfoForOrg(runner.Job.GithubJobId, runner.Job.Repository, targetConfig.GitHubToken);
 
                                 if (githubJob?.Status == "running" || githubJob?.Status == "in_progress")
                                 {