add instrumentation for tempo and pyroscope

Author: elasticroentgen

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base
 USER $APP_UID
 WORKDIR /app
 EXPOSE 8080
@@ -17,7 +17,31 @@ FROM build AS publish
 ARG BUILD_CONFIGURATION=Release
 RUN dotnet publish "GithubActionsOrchestrator.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
 
+# Download Pyroscope native profiler
+FROM base AS pyroscope-dl
+USER root
+ARG TARGETARCH
+RUN apt-get update && apt-get install -y --no-install-recommends wget && \
+    mkdir -p /pyroscope && \
+    PYRO_ARCH=$(if [ "$TARGETARCH" = "arm64" ]; then echo "aarch64"; else echo "x86_64"; fi) && \
+    wget -qO- "https://github.com/grafana/pyroscope-dotnet/releases/download/v0.14.1-pyroscope/pyroscope.0.14.1-glibc-${PYRO_ARCH}.tar.gz" | \
+    tar xz -C /pyroscope && \
+    apt-get remove -y wget && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
+
 FROM base AS final
 WORKDIR /app
 COPY --from=publish /app/publish .
+COPY --from=pyroscope-dl /pyroscope /pyroscope
+
+# Pyroscope CLR profiler
+ENV CORECLR_ENABLE_PROFILING=1
+ENV CORECLR_PROFILER={BD1A650D-AC5D-4896-B64F-D6FA25D6B26A}
+ENV CORECLR_PROFILER_PATH=/pyroscope/Pyroscope.Profiler.Native.so
+ENV LD_PRELOAD=/pyroscope/Pyroscope.Linux.ApiWrapper.x64.so
+ENV LD_LIBRARY_PATH=/pyroscope
+ENV DOTNET_EnableDiagnostics=1
+ENV DOTNET_EnableDiagnostics_IPC=0
+ENV DOTNET_EnableDiagnostics_Debugger=0
+ENV DOTNET_EnableDiagnostics_Profiler=1
+
 ENTRYPOINT ["dotnet", "GithubActionsOrchestrator.dll"]

GithubActionsOrchestrator.csproj

@@ -23,6 +23,12 @@
         <PackageReference Include="Serilog" Version="4.3.1" />
         <PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
         <PackageReference Include="Swashbuckle.AspNetCore" Version="10.1.2" />
+        <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.*" />
+        <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.*" />
+        <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.*" />
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.*" />
+        <PackageReference Include="Pyroscope" Version="0.*" />
+        <PackageReference Include="Pyroscope.OpenTelemetry" Version="0.*" />
     </ItemGroup>
 
     <ItemGroup>

PoolManager.cs

@@ -1,5 +1,7 @@
+using System.Diagnostics;
 using System.Reflection.Metadata.Ecma335;
 using GithubActionsOrchestrator.CloudControllers;
+
 using GithubActionsOrchestrator.Database;
 using GithubActionsOrchestrator.GitHub;
 using GithubActionsOrchestrator.Models;
@@ -257,8 +259,10 @@ private async Task CleanupDatabase()
 
     private async Task CheckForStuckRunners(List<GithubTargetConfiguration> targetConfig)
     {
+        using var activity = Program.OrchestratorActivitySource.StartActivity("maintenance.check_stuck_runners");
+
         // check the database for runners that are in "created" state for more then 5 minutes.
-        
+
         await using var db = new ActionsRunnerContext();
         var cutoffTime = DateTime.UtcNow - TimeSpan.FromMinutes(10);
 
@@ -275,9 +279,11 @@ private async Task CheckForStuckRunners(List<GithubTargetConfiguration> targetCo
             .Select(r => new { r.RunnerId, r.CloudServerId, r.Hostname, r.Cloud })
             .ToListAsync();
 
+        activity?.SetTag("stuck_runners.count", stuckRunners.Count);
+
         if (stuckRunners.Count == 0)
             return;
-        
+
         // Process stuck runners and create lifecycle entries
         var lifecycleEntries = new List<RunnerLifecycle>();
 
@@ -506,6 +512,8 @@ private async Task StartPoolRunners(List<GithubTargetConfiguration> targetConfig
 
     private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfig)
     {
+        using var activity = Program.OrchestratorActivitySource.StartActivity("maintenance.check_stuck_jobs");
+
         await using var db = new ActionsRunnerContext();
         var stuckTime = DateTime.UtcNow - TimeSpan.FromMinutes(10);
 
@@ -514,6 +522,8 @@ private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfi
             .Where(x => (x.State == JobState.Queued || x.State == JobState.Throttled) && x.RunnerId == null && x.QueueTime < stuckTime)
             .ToListAsync();
 
+        activity?.SetTag("stuck_jobs.count", stuckJobs.Count);
+
         foreach (var stuckJob in stuckJobs)
         {
             var owner = targetConfig.FirstOrDefault(x => x.Name == stuckJob.Owner);
@@ -893,9 +903,13 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
 
     private async Task<bool> DeleteRunner(DeleteRunnerTask rt)
     {
+        using var activity = Program.OrchestratorActivitySource.StartActivity("runner.delete");
+        activity?.SetTag("runner.db_id", rt.RunnerDbId);
+        activity?.SetTag("runner.server_id", rt.ServerId);
+
         await using var db = new ActionsRunnerContext();
         var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.RunnerId == rt.RunnerDbId);
-        
+
         try
         {
             ICloudController cc = _cc.FirstOrDefault(x => x.CloudIdentifier == runner.Cloud);
@@ -917,6 +931,8 @@ private async Task<bool> DeleteRunner(DeleteRunnerTask rt)
         }
         catch (Exception ex)
         {
+            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
+            activity?.AddException(ex);
             SentrySdk.CaptureException(ex, scope =>
             {
                 scope.SetTag("server-id", rt.ServerId.ToString());
@@ -952,6 +968,10 @@ private async Task<bool> DeleteRunner(DeleteRunnerTask rt)
 
     private async Task<bool> CreateRunner(CreateRunnerTask rt)
     {
+        using var activity = Program.OrchestratorActivitySource.StartActivity("runner.create");
+        activity?.SetTag("runner.db_id", rt.RunnerDbId);
+        activity?.SetTag("runner.repo", rt.RepoName);
+
         await using var db = new ActionsRunnerContext();
         var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.RunnerId == rt.RunnerDbId);
 
@@ -1054,6 +1074,8 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
                 {
                     newRunner = await cc.CreateNewRunner(runner.Arch, runner.Size, runnerToken, targetName, runner.IsCustom, runner.Profile);
                     _logger.LogInformation($"New Runner {newRunner.Name} [{runner.Size} on {runner.Arch}] entering pool for {targetName}.");
+                    activity?.SetTag("runner.hostname", newRunner.Name);
+                    activity?.SetTag("runner.cloud", cc.CloudIdentifier);
                     MachineCreatedCount.Labels(runner.Owner, runner.Size).Inc();
 
                     runner.Hostname = newRunner.Name;
@@ -1092,6 +1114,8 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
         }
         catch (Exception ex)
         {
+            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
+            activity?.AddException(ex);
             SentrySdk.CaptureException(ex, scope =>
             {
                 scope.SetTag("runner-size", runner.Size);

Program.cs

@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using System.Security.Cryptography;
 using System.Text;
 using System.Text.Json;
@@ -8,6 +9,8 @@
 using GithubActionsOrchestrator.Models;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 using Prometheus;
 using Serilog;
 using Serilog.Events;
@@ -18,6 +21,9 @@ public class Program
 {
     public static AutoScalerConfiguration Config = new();
 
+    internal const string ServiceName = "github-actions-orchestrator";
+    internal static readonly ActivitySource OrchestratorActivitySource = new(ServiceName);
+
     private static readonly Counter ProcessedJobCount = Metrics
         .CreateCounter("github_autoscaler_jobs_processed", "Number of processed jobs", labelNames: ["org", "size"]);
 
@@ -95,6 +101,18 @@ public static void Main(string[] args)
 
         WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
         builder.Services.AddSerilog();
+        builder.Services.AddOpenTelemetry()
+            .ConfigureResource(resource => resource.AddService(serviceName: ServiceName))
+            .WithTracing(tracing =>
+            {
+                tracing
+                    .AddSource(ServiceName)
+                    .AddSource("Npgsql")
+                    .AddAspNetCoreInstrumentation()
+                    .AddHttpClientInstrumentation()
+                    .AddOtlpExporter()
+                    .AddProcessor(new Pyroscope.OpenTelemetry.PyroscopeSpanProcessor());
+            });
         builder.Services.AddSingleton<RunnerQueue>();
         builder.Services.AddHostedService<PoolManager>();
 
@@ -209,6 +227,8 @@ public static bool LoadConfiguration()
 
     private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [FromServices] HetznerCloudController cloud, [FromServices] ILogger<Program> logger, [FromServices] RunnerQueue poolMgr)
     {
+        using var activity = OrchestratorActivitySource.StartActivity("webhook.github");
+
         // Verify webhook HMAC
         request.EnableBuffering();
         string requestBody;
@@ -227,6 +247,7 @@ private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [Fr
             }
             catch (Exception ex)
             {
+                activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
                 logger.LogWarning($"Webhook signature verification failed: {ex.Message}");
                 return Results.StatusCode(401);
             }
@@ -246,6 +267,8 @@ private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [Fr
             return Results.StatusCode(201);
         }
 
+        activity?.SetTag("github.action", action);
+
         if (!json.RootElement.TryGetProperty("workflow_job", out JsonElement workflowJson))
         {
             logger.LogDebug("Received a non-workflowJob request. Ignoring.");
@@ -274,6 +297,9 @@ private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [Fr
         string orgName = Config.TargetConfigs.FirstOrDefault(x => x.Target == TargetType.Organization && x.Name.ToLower() == orgNameRequest.ToLower())?.Name ?? orgNameRequest;
         string repoName = Config.TargetConfigs.FirstOrDefault(x => x.Target == TargetType.Repository && x.Name.ToLower() == repoNameRequest.ToLower())?.Name ?? repoNameRequest;
 
+        activity?.SetTag("github.job_id", jobId);
+        activity?.SetTag("github.repo", repoName);
+        activity?.SetTag("github.org", orgName);
 
         // Check if its an org or a repo
         if (String.IsNullOrEmpty(orgName))
@@ -292,81 +318,90 @@ private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [Fr
             return Results.StatusCode(201);
         }
 
-        
+
         await using var db = new ActionsRunnerContext();
 
         try
         {
             switch (action)
             {
                 case "queued":
-                    await JobQueued(logger, repoName, labels, orgName, poolMgr, isRepo ? TargetType.Repository : TargetType.Organization, jobId, jobUrl);
+                    using (OrchestratorActivitySource.StartActivity("webhook.github.queued"))
+                    {
+                        await JobQueued(logger, repoName, labels, orgName, poolMgr, isRepo ? TargetType.Repository : TargetType.Organization, jobId, jobUrl);
+                    }
                     break;
                 case "in_progress":
-                    var dbWorkflow = await db.Jobs.FirstOrDefaultAsync(x => x.GithubJobId == jobId);
-                    if (dbWorkflow == null)
+                    using (OrchestratorActivitySource.StartActivity("webhook.github.in_progress"))
                     {
-                        logger.LogWarning("Processing job on manually created runner");
-                        Job progressJob = new()
+                        var dbWorkflow = await db.Jobs.FirstOrDefaultAsync(x => x.GithubJobId == jobId);
+                        if (dbWorkflow == null)
                         {
-                            GithubJobId = jobId,
-                            Repository = repoName,
-                            Owner = isRepo ? repoName : orgName,
-                            State = JobState.InProgress,
-                            InProgressTime = DateTime.UtcNow,
-                            JobUrl = jobUrl,
-                            Orphan = true
-                        };
-                        await db.Jobs.AddAsync(progressJob);
-                    }
-                    else
-                    {
-                        dbWorkflow.State = JobState.InProgress;
-                        dbWorkflow.QueueTime = DateTime.UtcNow;
+                            logger.LogWarning("Processing job on manually created runner");
+                            Job progressJob = new()
+                            {
+                                GithubJobId = jobId,
+                                Repository = repoName,
+                                Owner = isRepo ? repoName : orgName,
+                                State = JobState.InProgress,
+                                InProgressTime = DateTime.UtcNow,
+                                JobUrl = jobUrl,
+                                Orphan = true
+                            };
+                            await db.Jobs.AddAsync(progressJob);
+                        }
+                        else
+                        {
+                            dbWorkflow.State = JobState.InProgress;
+                            dbWorkflow.QueueTime = DateTime.UtcNow;
+                        }
+                        await db.SaveChangesAsync();
+                        await JobInProgress(workflowJson, logger, jobId, repoName, orgName);
                     }
-                    await db.SaveChangesAsync();
-                    await JobInProgress(workflowJson, logger, jobId, repoName, orgName);
                     break;
                 case "completed":
-                    string conclusion = String.Empty;
-                    if (json.RootElement.TryGetProperty("conclusion", out JsonElement conclusionJson))
+                    using (OrchestratorActivitySource.StartActivity("webhook.github.completed"))
                     {
-                        conclusion = conclusionJson.GetString() ?? string.Empty;
-                    }
+                        string conclusion = String.Empty;
+                        if (json.RootElement.TryGetProperty("conclusion", out JsonElement conclusionJson))
+                        {
+                            conclusion = conclusionJson.GetString() ?? string.Empty;
+                        }
 
-                    var dbWorkflowComplete = await db.Jobs.FirstOrDefaultAsync(x => x.GithubJobId == jobId);
-                    if (dbWorkflowComplete == null)
-                    {
-                        logger.LogWarning($"Completed webhook for unknown job {jobId} in {repoName}. Creating record.");
-                        dbWorkflowComplete = new Job
+                        var dbWorkflowComplete = await db.Jobs.FirstOrDefaultAsync(x => x.GithubJobId == jobId);
+                        if (dbWorkflowComplete == null)
                         {
-                            GithubJobId = jobId,
-                            Repository = repoName,
-                            Owner = isRepo ? repoName : orgName,
-                            State = JobState.Completed,
-                            CompleteTime = DateTime.UtcNow,
-                            Orphan = true
-                        };
-                        await db.Jobs.AddAsync(dbWorkflowComplete);
-                        await db.SaveChangesAsync();
-                        return Results.StatusCode(201);
-                    }
-                    dbWorkflowComplete.CompleteTime = DateTime.UtcNow;
-                    bool wasCancelled = false;
-                    switch (conclusion)
-                    {
-                        case "cancelled":
-                            dbWorkflowComplete.State = JobState.Cancelled;
-                            await db.SaveChangesAsync();
-                            wasCancelled = true;
-                            break;
-                        default:
-                            dbWorkflowComplete.State = JobState.Completed;
+                            logger.LogWarning($"Completed webhook for unknown job {jobId} in {repoName}. Creating record.");
+                            dbWorkflowComplete = new Job
+                            {
+                                GithubJobId = jobId,
+                                Repository = repoName,
+                                Owner = isRepo ? repoName : orgName,
+                                State = JobState.Completed,
+                                CompleteTime = DateTime.UtcNow,
+                                Orphan = true
+                            };
+                            await db.Jobs.AddAsync(dbWorkflowComplete);
                             await db.SaveChangesAsync();
-                            break;
-                    }
+                            return Results.StatusCode(201);
+                        }
+                        dbWorkflowComplete.CompleteTime = DateTime.UtcNow;
+                        bool wasCancelled = false;
+                        switch (conclusion)
+                        {
+                            case "cancelled":
+                                dbWorkflowComplete.State = JobState.Cancelled;
+                                await db.SaveChangesAsync();
+                                wasCancelled = true;
+                                break;
+                            default:
+                                dbWorkflowComplete.State = JobState.Completed;
+                                await db.SaveChangesAsync();
+                                break;
+                        }
 
-                    await JobCompleted(logger, jobId, poolMgr, repoName, orgName, workflowJson, wasCancelled);
+                        await JobCompleted(logger, jobId, poolMgr, repoName, orgName, workflowJson, wasCancelled);
+                    }
                     break;
                 default:
                     logger.LogWarning("Unknown action. Ignoring");
@@ -375,12 +410,14 @@ private static async Task<IResult> GithubWebhookHandler(HttpRequest request, [Fr
         }
         catch (Exception ex)
         {
+            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
+            activity?.AddException(ex);
             // This should make the webhook as bad and the timer will redeliver it after a while
             Log.Error($"Failed to process {action} webhook: {ex.Message}");
             return Results.StatusCode(500);
         }
 
-        // All was well 
+        // All was well
         return Results.StatusCode(201);
     }