add better exception logging

Author: elasticroentgen

CloudControllers/ProxmoxCloudController.cs

@@ -53,22 +53,13 @@ public async Task<Machine> CreateNewRunner(string arch, string size, string runn
         // Get next available VMID
 
         await _semaphore.WaitAsync();
-
         string macaddress = string.Empty;
-        
-        
-        
         int newVmId;
+        
         try
         {
-            
             // Select node
             var resources = (await client.Cluster.Resources.GetAsync("vm")).ToList();
-            /*var vmCountByNode = resources
-                .Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix))
-                .GroupBy(x => x.Node)
-                .Select(x => new { Node = x.Key, Count = x.Count()});*/
-
             var availableNodes = resources.Select(x => x.Node).Distinct().ToList();
 
             var vmCountByNode = availableNodes

ExceptionExtensions.cs

@@ -0,0 +1,33 @@
+// Create an extension method for formatting exception details
+
+using System.Text;
+
+public static class ExceptionExtensions
+{
+    public static string GetFullExceptionDetails(this Exception ex)
+    {
+        var sb = new StringBuilder();
+        var currentEx = ex;
+        var level = 0;
+        
+        while (currentEx != null)
+        {
+            if (level > 0)
+            {
+                sb.AppendLine($"Inner Exception Level {level}:");
+            }
+            sb.AppendLine($"Message: {currentEx.Message}");
+            sb.AppendLine($"Stack Trace: {currentEx.StackTrace}");
+            
+            currentEx = currentEx.InnerException;
+            level++;
+            
+            if (currentEx != null)
+            {
+                sb.AppendLine();
+            }
+        }
+        
+        return sb.ToString();
+    }
+}

PoolManager.cs

@@ -188,9 +188,9 @@ private async Task ProcessStats(List<GithubTargetConfiguration> targetConfig)
             {
                 CspRunnerCount.Labels(cc.CloudIdentifier).Set(await cc.GetServerCountFromCsp());
             }
-            catch
+            catch(Exception ex)
             {
-                _logger.LogWarning($"Unable to get runner count from CSP {cc.CloudIdentifier}");
+                _logger.LogWarning($"Unable to get runner count from CSP {cc.CloudIdentifier}: {ex.GetFullExceptionDetails()}");
             }
         }
 
@@ -723,7 +723,7 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
         }
 
         var cc = _cc.First(x => x.CloudIdentifier == selectedProvider.Cloud);
-        
+
         try
         {
             string targetName = rt.TargetType switch
@@ -732,34 +732,57 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
                 TargetType.Organization => runner.Owner,
                 _ => throw new ArgumentOutOfRangeException()
             };
-            
-            
-            Machine newRunner = await cc.CreateNewRunner(runner.Arch, runner.Size, rt.RunnerToken, targetName, runner.IsCustom, runner.Profile);
-            _logger.LogInformation($"New Runner {newRunner.Name} [{runner.Size} on {runner.Arch}] entering pool for {targetName}.");
-            MachineCreatedCount.Labels(runner.Owner, runner.Size).Inc();
-
-            runner.Hostname = newRunner.Name;
-            runner.IsOnline = true;
-            runner.CloudServerId = newRunner.Id;
-            runner.IPv4 = newRunner.Ipv4;
-            runner.Cloud = cc.CloudIdentifier;
-            runner.ProvisionId = newRunner.ProvisionId;
-            runner.ProvisionPayload = newRunner.ProvisionPayload;
-            
-            
-            runner.Lifecycle.Add(new RunnerLifecycle
+
+
+            Machine newRunner;
+            int retryAttempt = 0;
+            const int maxRetries = 1;
+            const int retryDelayMs = 1000; // 1 second delay between retries
+
+            while (retryAttempt <= maxRetries)
             {
-                Status = RunnerStatus.Created,
-                EventTimeUtc = DateTime.UtcNow,
-                Event = $"New Runner {newRunner.Name} [{runner.Size} on {runner.Arch}] entering pool for {targetName}."
-            });
+
+                try
+                {
+                    newRunner = await cc.CreateNewRunner(runner.Arch, runner.Size, rt.RunnerToken, targetName, runner.IsCustom, runner.Profile);
+                    _logger.LogInformation($"New Runner {newRunner.Name} [{runner.Size} on {runner.Arch}] entering pool for {targetName}.");
+                    MachineCreatedCount.Labels(runner.Owner, runner.Size).Inc();
+
+                    runner.Hostname = newRunner.Name;
+                    runner.IsOnline = true;
+                    runner.CloudServerId = newRunner.Id;
+                    runner.IPv4 = newRunner.Ipv4;
+                    runner.Cloud = cc.CloudIdentifier;
+                    runner.ProvisionId = newRunner.ProvisionId;
+                    runner.ProvisionPayload = newRunner.ProvisionPayload;
+                    runner.Lifecycle.Add(new RunnerLifecycle
+                    {
+                        Status = RunnerStatus.Created,
+                        EventTimeUtc = DateTime.UtcNow,
+                        Event = $"New Runner {newRunner.Name} [{runner.Size} on {runner.Arch}] entering pool for {targetName}."
+                    });
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    if (retryAttempt == maxRetries)
+                    {
+                        _logger.LogError(ex, $"Failed to create runner after {maxRetries + 1} attempts");
+                        throw; // Re-throw the exception after all retries are exhausted
+                    }
+        
+                    _logger.LogWarning(ex, $"Failed to create runner (attempt {retryAttempt + 1}/{maxRetries + 1}). Retrying...");
+                    await Task.Delay(retryDelayMs);
+                    retryAttempt++; 
+                }
+            }
             await db.SaveChangesAsync();
 
             return true;
         }
         catch (Exception ex)
         {
-            _logger.LogError($"Unable to create runner [{runner.Size} on {runner.Arch} | Retry: {rt.RetryCount}]: {ex.Message}");
+            _logger.LogError($"Unable to create runner [{runner.Size} on {runner.Arch} | Retry: {rt.RetryCount}]: {ex.GetFullExceptionDetails()}");
             runner.Lifecycle.Add(new RunnerLifecycle
             {
                 Status = RunnerStatus.Failure,
@@ -789,7 +812,7 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
             {
                 Cloud = cc.CloudIdentifier,
                 Size = runner.Size,
-                UnbanTime = DateTime.UtcNow + TimeSpan.FromMinutes(30) 
+                UnbanTime = DateTime.UtcNow + TimeSpan.FromMinutes(10) 
             });
 
             return false;