From 73a30eb5bdac50d9553fb1a890f9ac9af0e56ca0 Mon Sep 17 00:00:00 2001 From: Amir Szekely Date: Thu, 28 Aug 2025 11:34:10 -0400 Subject: [PATCH] wip --- src/runner.ts | 76 +++++++++++++++++++ .../github-runners-test.assets.json | 4 +- .../github-runners-test.template.json | 58 ++++++++++++++ test/default.integ.ts | 1 + 4 files changed, 137 insertions(+), 2 deletions(-) diff --git a/src/runner.ts b/src/runner.ts index 5dea31ce..a64572af 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -883,4 +883,80 @@ export class GitHubRunners extends Construct implements ec2.IConnectable { }), }); } + + /** + * Creates a CloudWatch dashboard with key metrics for GitHubRunners. + * + * @param name The name of the dashboard. + */ + public createDashboard(name: string = 'GitHub-Runners'): cloudwatch.Dashboard { + const dashboard = new cloudwatch.Dashboard(this, 'Dashboard', { + dashboardName: name, + }); + + dashboard.addWidgets( + new cloudwatch.GraphWidget({ + title: 'Successful jobs by label', + left: this.providers.map(p => new cloudwatch.Metric({ + namespace: 'GitHubRunners', + metricName: 'JobCompleted', + dimensionsMap: { + ProviderLabels: p.labels.join(','), + Status: 'Succeeded', + }, + label: p.labels.join(', '), + statistic: cloudwatch.Stats.SUM, + })), + stacked: true, + view: cloudwatch.GraphWidgetView.TIME_SERIES, + legendPosition: cloudwatch.LegendPosition.BOTTOM, + }), + new cloudwatch.GraphWidget({ + title: 'Failed jobs by label and failure', + left: this.providers.map(p => new cloudwatch.Metric({ + namespace: 'GitHubRunners', + metricName: 'JobCompleted', + dimensionsMap: { + ProviderLabels: p.labels.join(','), + }, + label: p.labels.join(', '), + statistic: cloudwatch.Stats.SUM, + })), + stacked: true, + view: cloudwatch.GraphWidgetView.TIME_SERIES, + legendPosition: cloudwatch.LegendPosition.BOTTOM, + }), + new cloudwatch.GraphWidget({ + title: 'Runner Succeeded', + left: [this.metricSucceeded()], + }), + new cloudwatch.GraphWidget({ + title: 'Runner Failed', + left: [this.metricFailed()], + }), + new cloudwatch.GraphWidget({ + title: 'Runner Execution Time (ms)', + left: [this.metricTime()], + }), + ); + + dashboard.addWidgets(new cloudwatch.LogQueryWidget({ + logGroupNames: [this.webhook.handler.logGroup.logGroupName], + title: 'Recent Webhook Errors', + view: cloudwatch.LogQueryVisualizationType.TABLE, + queryString: new logs.QueryString({ + filterStatements: [ + `strcontains(@logStream, "${this.webhook.handler.functionName}")`, + 'level = "ERROR"', + ], + sort: '@timestamp desc', + fields: ['@timestamp', 'message'], + limit: 20, + }).toString(), + width: 12, + height: 6, + })); + + return dashboard; + } } diff --git a/test/default.integ.snapshot/github-runners-test.assets.json b/test/default.integ.snapshot/github-runners-test.assets.json index d44dcebd..49aa17ba 100644 --- a/test/default.integ.snapshot/github-runners-test.assets.json +++ b/test/default.integ.snapshot/github-runners-test.assets.json @@ -222,7 +222,7 @@ } } }, - "3b80f113bf1080dd8be08e53a2fe41d076fb71707d73d9bec289feb062840add": { + "edb5253e94f2be0eaa455761183dceb11206dfb3cc59a38ae76d6d02f3cb31d9": { "source": { "path": "github-runners-test.template.json", "packaging": "file" @@ -230,7 +230,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "3b80f113bf1080dd8be08e53a2fe41d076fb71707d73d9bec289feb062840add.json", + "objectKey": "edb5253e94f2be0eaa455761183dceb11206dfb3cc59a38ae76d6d02f3cb31d9.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/test/default.integ.snapshot/github-runners-test.template.json b/test/default.integ.snapshot/github-runners-test.template.json index 8fbf9361..12381acc 100644 --- a/test/default.integ.snapshot/github-runners-test.template.json +++ b/test/default.integ.snapshot/github-runners-test.template.json @@ -21402,6 +21402,64 @@ "QueryString": "fields @timestamp, message.notice, message.deliveryId, message.guid\n| filter isPresent(message.deliveryId)\n| sort @timestamp desc\n| limit 100" } }, + "runnersDashboardAD2609CF": { + "Type": "AWS::CloudWatch::Dashboard", + "Properties": { + "DashboardBody": { + "Fn::Join": [ + "", + [ + "{\"widgets\":[{\"type\":\"metric\",\"width\":6,\"height\":6,\"x\":0,\"y\":0,\"properties\":{\"view\":\"timeSeries\",\"title\":\"Successful jobs by label\",\"region\":\"", + { + "Ref": "AWS::Region" + }, + "\",\"stacked\":true,\"metrics\":[[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild-x64\",\"Status\",\"Succeeded\",{\"label\":\"codebuild-x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild-ubuntu-2404-x64\",\"Status\",\"Succeeded\",{\"label\":\"codebuild-ubuntu-2404-x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild,linux,arm64\",\"Status\",\"Succeeded\",{\"label\":\"codebuild, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild,windows,x64\",\"Status\",\"Succeeded\",{\"label\":\"codebuild, windows, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,linux,x64\",\"Status\",\"Succeeded\",{\"label\":\"ecs, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,ubuntu-2404,x64\",\"Status\",\"Succeeded\",{\"label\":\"ecs, ubuntu-2404, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,linux,arm64\",\"Status\",\"Succeeded\",{\"label\":\"ecs, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,windows,x64\",\"Status\",\"Succeeded\",{\"label\":\"ecs, windows, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"lambda,x64\",\"Status\",\"Succeeded\",{\"label\":\"lambda, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"lambda,arm64\",\"Status\",\"Succeeded\",{\"label\":\"lambda, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate,linux,x64\",\"Status\",\"Succeeded\",{\"label\":\"fargate, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate-spot,linux,x64\",\"Status\",\"Succeeded\",{\"label\":\"fargate-spot, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate,linux,arm64\",\"Status\",\"Succeeded\",{\"label\":\"fargate, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate-spot,linux,arm64\",\"Status\",\"Succeeded\",{\"label\":\"fargate-spot, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate,windows,x64\",\"Status\",\"Succeeded\",{\"label\":\"fargate, windows, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2,linux,x64\",\"Status\",\"Succeeded\",{\"label\":\"ec2, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2-spot,linux,x64\",\"Status\",\"Succeeded\",{\"label\":\"ec2-spot, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2,linux,arm64\",\"Status\",\"Succeeded\",{\"label\":\"ec2, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2,windows,x64\",\"Status\",\"Succeeded\",{\"label\":\"ec2, windows, x64\",\"stat\":\"Sum\"}]],\"yAxis\":{},\"legend\":{\"position\":\"bottom\"}}},{\"type\":\"metric\",\"width\":6,\"height\":6,\"x\":6,\"y\":0,\"properties\":{\"view\":\"timeSeries\",\"title\":\"Failed jobs by label and failure\",\"region\":\"", + { + "Ref": "AWS::Region" + }, + "\",\"stacked\":true,\"metrics\":[[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild-x64\",{\"label\":\"codebuild-x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild-ubuntu-2404-x64\",{\"label\":\"codebuild-ubuntu-2404-x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild,linux,arm64\",{\"label\":\"codebuild, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"codebuild,windows,x64\",{\"label\":\"codebuild, windows, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,linux,x64\",{\"label\":\"ecs, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,ubuntu-2404,x64\",{\"label\":\"ecs, ubuntu-2404, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,linux,arm64\",{\"label\":\"ecs, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ecs,windows,x64\",{\"label\":\"ecs, windows, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"lambda,x64\",{\"label\":\"lambda, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"lambda,arm64\",{\"label\":\"lambda, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate,linux,x64\",{\"label\":\"fargate, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate-spot,linux,x64\",{\"label\":\"fargate-spot, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate,linux,arm64\",{\"label\":\"fargate, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate-spot,linux,arm64\",{\"label\":\"fargate-spot, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"fargate,windows,x64\",{\"label\":\"fargate, windows, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2,linux,x64\",{\"label\":\"ec2, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2-spot,linux,x64\",{\"label\":\"ec2-spot, linux, x64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2,linux,arm64\",{\"label\":\"ec2, linux, arm64\",\"stat\":\"Sum\"}],[\"GitHubRunners\",\"JobCompleted\",\"ProviderLabels\",\"ec2,windows,x64\",{\"label\":\"ec2, windows, x64\",\"stat\":\"Sum\"}]],\"yAxis\":{},\"legend\":{\"position\":\"bottom\"}}},{\"type\":\"metric\",\"width\":6,\"height\":6,\"x\":12,\"y\":0,\"properties\":{\"view\":\"timeSeries\",\"title\":\"Runner Succeeded\",\"region\":\"", + { + "Ref": "AWS::Region" + }, + "\",\"metrics\":[[\"AWS/States\",\"ExecutionsSucceeded\",\"StateMachineArn\",\"", + { + "Ref": "runnersRunnerOrchestratorF9B66EBA" + }, + "\",{\"stat\":\"Sum\"}]],\"yAxis\":{}}},{\"type\":\"metric\",\"width\":6,\"height\":6,\"x\":18,\"y\":0,\"properties\":{\"view\":\"timeSeries\",\"title\":\"Runner Failed\",\"region\":\"", + { + "Ref": "AWS::Region" + }, + "\",\"metrics\":[[\"AWS/States\",\"ExecutionsFailed\",\"StateMachineArn\",\"", + { + "Ref": "runnersRunnerOrchestratorF9B66EBA" + }, + "\",{\"stat\":\"Sum\"}]],\"yAxis\":{}}},{\"type\":\"metric\",\"width\":6,\"height\":6,\"x\":0,\"y\":6,\"properties\":{\"view\":\"timeSeries\",\"title\":\"Runner Execution Time (ms)\",\"region\":\"", + { + "Ref": "AWS::Region" + }, + "\",\"metrics\":[[\"AWS/States\",\"ExecutionTime\",\"StateMachineArn\",\"", + { + "Ref": "runnersRunnerOrchestratorF9B66EBA" + }, + "\"]],\"yAxis\":{}}},{\"type\":\"log\",\"width\":12,\"height\":6,\"x\":0,\"y\":12,\"properties\":{\"view\":\"table\",\"title\":\"Recent Webhook Errors\",\"region\":\"", + { + "Ref": "AWS::Region" + }, + "\",\"query\":\"SOURCE '", + { + "Ref": "OrchestratorLogFB9610E7" + }, + "' | fields @timestamp, message\\n| filter strcontains(@logStream, \\\"", + { + "Ref": "runnersWebhookHandlerwebhookhandler22779A81" + }, + "\\\")\\n| filter level = \\\"ERROR\\\"\\n| sort @timestamp desc\\n| limit 20\"}}]}" + ] + ] + }, + "DashboardName": "GitHub-Runners" + } + }, "OrchestratorLogFB9610E7": { "Type": "AWS::Logs::LogGroup", "Properties": { diff --git a/test/default.integ.ts b/test/default.integ.ts index b3196730..25907a75 100644 --- a/test/default.integ.ts +++ b/test/default.integ.ts @@ -292,5 +292,6 @@ const runners = new GitHubRunners(stack, 'runners', { runners.metricJobCompleted(); runners.failedImageBuildsTopic(); runners.createLogsInsightsQueries(); +runners.createDashboard(); app.synth();