From 16029c7c671184aae97dcfb3ff2007b9709dce71 Mon Sep 17 00:00:00 2001 From: Missy Messa Date: Thu, 21 May 2026 12:29:27 -0700 Subject: [PATCH] Modify AlertHookController to create AzDO work items instead of GitHub issues Rewrites the Grafana alert webhook handler to create DNCENG Task work items in Azure DevOps (area path: .NET Engineering Services\First Responders) instead of opening GitHub issues. Key changes: - AlertHookController now uses IAzureDevOpsClient with Managed Identity auth - Removed Octokit/GitHub dependencies from the alert path - Alert fires: creates work item in Backlog state with 'Active Alert' tag - Alert resolves: moves work item to Done with 'Inactive Alert' tag - Reuses existing work items on recurrence (WIQL search by automation ID) - Added CreateWorkItemAsync, UpdateWorkItemAsync, AddWorkItemCommentAsync, and QueryWorkItemsAsync to IAzureDevOpsClient - New AzureDevOpsAlertOptions config section ('AzureDevOpsAlert') - Updated unit tests to match new controller signature Resolves: https://dnceng.visualstudio.com/internal/_workitems/edit/8579 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AlertHookControllerTests.cs | 109 +++-- .../.config/settings.Production.json | 4 + .../.config/settings.Staging.json | 4 + .../DotNet.Status.Web/.config/settings.json | 8 + .../Controllers/AlertHookController.cs | 407 ++++++++---------- .../DotNet.Status.Web.csproj | 4 + .../Options/AzureDevOpsAlertOptions.cs | 16 + .../DotNet.Status.Web/Startup.cs | 1 + .../AzureDevOpsClient/AzureDataTypes.cs | 16 + .../AzureDevOpsClient/AzureDevOpsClient.cs | 165 +++++++ .../AzureDevOpsClient/IAzureDevOpsClient.cs | 4 + .../MockAzureClient.cs | 22 +- .../MockTimeoutAzureClient.cs | 20 + 13 files changed, 507 insertions(+), 273 deletions(-) create mode 100644 src/DotNet.Status.Web/DotNet.Status.Web/Options/AzureDevOpsAlertOptions.cs diff --git a/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AlertHookControllerTests.cs b/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AlertHookControllerTests.cs index ffdaedffa..7138c537d 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AlertHookControllerTests.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AlertHookControllerTests.cs @@ -1,15 +1,16 @@ using System; -using System.Reflection; +using System.Collections.Generic; +using System.Collections.Immutable; using AwesomeAssertions; using DotNet.Status.Web.Controllers; using DotNet.Status.Web.Models; using DotNet.Status.Web.Options; -using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.DotNet.Internal.AzureDevOps; +using Microsoft.DotNet.Internal.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Moq; using NUnit.Framework; -using Octokit; namespace DotNet.Status.Web.Tests; @@ -17,9 +18,9 @@ namespace DotNet.Status.Web.Tests; public class AlertHookControllerTests { [Test] - public void GenerateNewIssue_WithMissingEvalMatchesAndNotificationTargets_DoesNotThrow() + public void GenerateDescription_WithMissingEvalMatches_DoesNotThrow() { - AlertHookController controller = CreateController(null); + AlertHookController controller = CreateController(); GrafanaNotification notification = new GrafanaNotification { Title = "Alert title", @@ -29,19 +30,19 @@ public void GenerateNewIssue_WithMissingEvalMatchesAndNotificationTargets_DoesNo EvalMatches = null, }; - Action action = () => InvokeGenerateNewIssue(controller, notification); + Action action = () => controller.GenerateDescription(notification); action.Should().NotThrow(); - NewIssue issue = InvokeGenerateNewIssue(controller, notification); - issue.Body.Should().Contain("Please investigate"); - issue.Body.Should().NotContain(", please investigate"); + string description = controller.GenerateDescription(notification); + description.Should().Contain("Supplemental text"); + description.Should().Contain("Grafana-Automated-Alert-Id-"); } [Test] - public void GenerateNewNotificationComment_WithMissingEvalMatches_DoesNotThrow() + public void GenerateComment_WithMissingEvalMatches_DoesNotThrow() { - AlertHookController controller = CreateController(Array.Empty()); + AlertHookController controller = CreateController(); GrafanaNotification notification = new GrafanaNotification { Title = "Alert title", @@ -51,31 +52,65 @@ public void GenerateNewNotificationComment_WithMissingEvalMatches_DoesNotThrow() EvalMatches = null, }; - Action action = () => InvokeGenerateNewNotificationComment(controller, notification); + Action action = () => controller.GenerateComment(notification); action.Should().NotThrow(); - string comment = InvokeGenerateNewNotificationComment(controller, notification); - comment.Should().Contain("Metric state changed to *alerting*"); + string comment = controller.GenerateComment(notification); + comment.Should().Contain("Metric state changed to"); + comment.Should().Contain("alerting"); } - private static AlertHookController CreateController(string[] notificationTargets) + [Test] + public void GenerateTitle_WithPrefix_PrependsPrefixToTitle() { - Mock tokenProvider = new(MockBehavior.Strict); - IOptions githubOptions = Microsoft.Extensions.Options.Options.Create(new GitHubConnectionOptions + AlertHookController controller = CreateController(); + GrafanaNotification notification = new GrafanaNotification { - Organization = "dotnet", - Repository = "dnceng", - NotificationTargets = notificationTargets, - AlertLabels = Array.Empty(), - EnvironmentLabels = Array.Empty(), + Title = "CPU High", + State = "alerting", + }; + + string title = controller.GenerateTitle(notification); + + title.Should().Be("[test] CPU High"); + } + + [Test] + public void GenerateDescription_WithEvalMatches_IncludesMetrics() + { + AlertHookController controller = CreateController(); + GrafanaNotification notification = new GrafanaNotification + { + Title = "Alert title", + State = "alerting", + Message = "High CPU", + RuleUrl = "https://example/rule", + EvalMatches = new List + { + new GrafanaNotificationMatch { Metric = "cpu_usage", Value = 95.5 }, + }.ToImmutableList(), + }; + + string description = controller.GenerateDescription(notification); + + description.Should().Contain("cpu_usage"); + description.Should().Contain("95.5"); + } + + private static AlertHookController CreateController() + { + Mock azureDevOpsClient = new(MockBehavior.Strict); + Mock> clientFactory = new(MockBehavior.Strict); + IOptions alertOptions = Microsoft.Extensions.Options.Options.Create(new AzureDevOpsAlertOptions + { + Organization = "dnceng", + Project = "internal", + AreaPath = @"internal\.NET Engineering Services\First Responders", + WorkItemType = "DNCENG Task", TitlePrefix = "[test] ", SupplementalBodyText = "Supplemental text", }); - IOptions clientOptions = Microsoft.Extensions.Options.Options.Create(new GitHubClientOptions - { - ProductHeader = new ProductHeaderValue("DotNetStatusWebTests"), - }); IOptions grafanaOptions = Microsoft.Extensions.Options.Options.Create(new GrafanaOptions { @@ -83,26 +118,10 @@ private static AlertHookController CreateController(string[] notificationTargets }); return new AlertHookController( - tokenProvider.Object, - githubOptions, - clientOptions, + clientFactory.Object, + alertOptions, grafanaOptions, NullLogger.Instance); } - - private static NewIssue InvokeGenerateNewIssue(AlertHookController controller, GrafanaNotification notification) - { - MethodInfo method = typeof(AlertHookController).GetMethod("GenerateNewIssue", BindingFlags.Instance | BindingFlags.NonPublic); - method.Should().NotBeNull(); - - return (NewIssue)method.Invoke(controller, new object[] { notification }); - } - - private static string InvokeGenerateNewNotificationComment(AlertHookController controller, GrafanaNotification notification) - { - MethodInfo method = typeof(AlertHookController).GetMethod("GenerateNewNotificationComment", BindingFlags.Instance | BindingFlags.NonPublic); - method.Should().NotBeNull(); - - return (string)method.Invoke(controller, new object[] { notification }); - } } + diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json index 30a850b7a..a541d0ada 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json @@ -12,6 +12,10 @@ "EnvironmentLabels": [ "Production" ], "TitlePrefix": "Production - " }, + "AzureDevOpsAlert": { + "TitlePrefix": "Production - ", + "SupplementalBodyText": "This alert was auto-generated by Grafana monitoring in the Production environment." + }, "DataProtection": { "KeyBlobUri": "https://dotnetengstatusprod.blob.core.windows.net/site/keys.xml", "DataProtectionKeyUri": "https://dotneteng-status-prod.vault.azure.net/keys/dotnet-status-data-protection/" diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json index ce66a481b..b9d6857ea 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json @@ -11,6 +11,10 @@ "EnvironmentLabels": [ "Staging" ], "TitlePrefix": "Staging - " }, + "AzureDevOpsAlert": { + "TitlePrefix": "Staging - ", + "SupplementalBodyText": "This alert was auto-generated by Grafana monitoring in the Staging environment." + }, "DataProtection": { "KeyBlobUri": "https://dotnetengstatusstaging.blob.core.windows.net/site/keys.xml", "DataProtectionKeyUri": "https://dotneteng-status-staging.vault.azure.net/keys/dotnet-status-data-protection/" diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json index de757bb26..e8b728928 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json @@ -285,6 +285,14 @@ "RcaRequestedLabels": [ "RCA Requested" ], "RcaLabel": "RCA" }, + "AzureDevOpsAlert": { + "Organization": "dnceng", + "Project": "internal", + "AreaPath": "internal\\.NET Engineering Services\\First Responders", + "WorkItemType": "DNCENG Task", + "TitlePrefix": "", + "SupplementalBodyText": "" + }, "AzureTableTokenStore": { "TableName": "tokens" }, diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AlertHookController.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AlertHookController.cs index bd133cb28..82ed0a267 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AlertHookController.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AlertHookController.cs @@ -13,10 +13,10 @@ using DotNet.Status.Web.Options; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; -using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.DotNet.Internal.AzureDevOps; +using Microsoft.DotNet.Internal.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Octokit; namespace DotNet.Status.Web.Controllers; @@ -25,31 +25,25 @@ namespace DotNet.Status.Web.Controllers; [AllowAnonymous] public class AlertHookController : ControllerBase { - public const string NotificationIdLabel = "Grafana Alert"; - public const string ActiveAlertLabel = "Active Alert"; - public const string InactiveAlertLabel = "Inactive Alert"; + public const string NotificationTag = "Grafana Alert"; + public const string ActiveAlertTag = "Active Alert"; + public const string InactiveAlertTag = "Inactive Alert"; public const string BodyLabelTextFormat = "Grafana-Automated-Alert-Id-{0}"; public const string NotificationTagName = "NotificationId"; - private static bool s_labelsCreated; - private static readonly SemaphoreSlim s_labelLock = new SemaphoreSlim(1); - - private readonly IOptions _githubOptions; - private readonly IOptions _githubClientOptions; + private readonly IOptions _alertOptions; private readonly IOptions _grafanaOptions; private readonly ILogger _logger; - private readonly IGitHubTokenProvider _tokenProvider; + private readonly IClientFactory _azureDevOpsClientFactory; public AlertHookController( - IGitHubTokenProvider tokenProvider, - IOptions githubOptions, - IOptions githubClientOptions, + IClientFactory azureDevOpsClientFactory, + IOptions alertOptions, IOptions grafanaOptions, ILogger logger) { - _tokenProvider = tokenProvider; - _githubOptions = githubOptions; - _githubClientOptions = githubClientOptions; + _azureDevOpsClientFactory = azureDevOpsClientFactory; + _alertOptions = alertOptions; _grafanaOptions = grafanaOptions; _logger = logger; } @@ -79,133 +73,177 @@ public async Task NotifyAsync(GrafanaNotification notification) return NoContent(); } + private Reference GetAzureDevOpsClient() + { + return _azureDevOpsClientFactory.GetClient("dnceng"); + } + private async Task OpenNewNotificationAsync(GrafanaNotification notification) { - string org = _githubOptions.Value.Organization; - string repo = _githubOptions.Value.Repository; + AzureDevOpsAlertOptions options = _alertOptions.Value; _logger.LogInformation( - "Alert state detected for {ruleUrl} in stage {ruleState}, porting to github repo {org}/{repo}", + "Alert state detected for {ruleUrl} in stage {ruleState}, creating AzDO work item in {project}", notification.RuleUrl, notification.State, - org, - repo); + options.Project); - IGitHubClient client = await GetGitHubClientAsync(_githubOptions.Value.Organization, _githubOptions.Value.Repository); - Issue issue = await GetExistingIssueAsync(client, notification); - await EnsureLabelsAsync(client, org, repo); - if (issue == null) + using var clientRef = GetAzureDevOpsClient(); + var client = clientRef.Value; + + WorkItem existingWorkItem = await GetExistingWorkItemAsync(client, notification); + + if (existingWorkItem == null) { - _logger.LogInformation("No existing issue found, creating new active issue with {label}", - ActiveAlertLabel); - issue = await client.Issue.Create(org, repo, GenerateNewIssue(notification)); - _logger.LogInformation("Github issue {org}/{repo}#{issueNumber} created", org, repo, issue.Number); + _logger.LogInformation("No existing work item found, creating new active work item with tag {tag}", + ActiveAlertTag); + + var fields = new Dictionary + { + ["System.Title"] = GenerateTitle(notification), + ["System.Description"] = GenerateDescription(notification), + ["System.State"] = "Backlog", + ["System.Tags"] = BuildTagString(NotificationTag, ActiveAlertTag), + }; + + WorkItem workItem = await client.CreateWorkItemAsync( + options.Project, + options.WorkItemType, + fields, + options.AreaPath, + CancellationToken.None); + + _logger.LogInformation("AzDO work item {project}/{workItemId} created", + options.Project, workItem?.Id); } else { + int workItemId = existingWorkItem.Id; _logger.LogInformation( - "Found existing issue {org}/{repo}#{issueNumber}, replacing {inactiveTag} with {activeTag}", - org, - repo, - issue.Number, - InactiveAlertLabel, - ActiveAlertLabel); - - await GitHubModifications.TryRemoveAsync(() => client.Issue.Labels.RemoveFromIssue(org, repo, issue.Number, InactiveAlertLabel), _logger); - await GitHubModifications.TryCreateAsync(() => - client.Issue.Labels.AddToIssue(org, repo, issue.Number, new[] {ActiveAlertLabel}), - _logger); - - _logger.LogInformation("Adding recurrence comment to {org}/{repo}#{issueNumber}", - org, - repo, - issue.Number); - IssueComment comment = await client.Issue.Comment.Create(org, - repo, - issue.Number, - GenerateNewNotificationComment(notification)); - _logger.LogInformation("Created comment {org}/{repo}#{issue}-issuecomment-{comment}", - org, - repo, - issue.Id, - comment.Id); - } - } - - private string GenerateNewNotificationComment(GrafanaNotification notification) - { - string metricText = BuildMetricText(notification); - string icon = GetIcon(notification); - string image = !string.IsNullOrEmpty(notification.ImageUrl) ? $"![Metric Graph]({notification.ImageUrl})" : string.Empty; + "Found existing work item {workItemId}, updating tags and adding recurrence comment", + workItemId); - return $@":{icon}: Metric state changed to *{notification.State}* + string currentTags = GetFieldValue(existingWorkItem, "System.Tags") ?? ""; + string newTags = UpdateTags(currentTags, addTag: ActiveAlertTag, removeTag: InactiveAlertTag); -> {notification.Message?.Replace("\n", "\n> ")} - -{metricText} + var fields = new Dictionary + { + ["System.State"] = "Backlog", + ["System.Tags"] = newTags, + }; -{image} + await client.UpdateWorkItemAsync(options.Project, workItemId, fields, CancellationToken.None); + await client.AddWorkItemCommentAsync( + options.Project, + workItemId, + GenerateComment(notification), + CancellationToken.None); -[Go to rule]({notification.RuleUrl})".Replace("\r\n","\n"); + _logger.LogInformation("Updated work item {workItemId} with recurrence comment", workItemId); + } } - private NewIssue GenerateNewIssue(GrafanaNotification notification) + private async Task CloseExistingNotificationAsync(GrafanaNotification notification) { - string metricText = BuildMetricText(notification); - string icon = GetIcon(notification); - string image = !string.IsNullOrEmpty(notification.ImageUrl) ? $"![Metric Graph]({notification.ImageUrl})" : string.Empty; + AzureDevOpsAlertOptions options = _alertOptions.Value; + + using var clientRef = GetAzureDevOpsClient(); + var client = clientRef.Value; - string issueTitle = notification.Title; + WorkItem existingWorkItem = await GetExistingWorkItemAsync(client, notification); - GitHubConnectionOptions options = _githubOptions.Value; - string prefix = options.TitlePrefix; - if (prefix != null) + if (existingWorkItem == null) { - issueTitle = prefix + issueTitle; + _logger.LogInformation("No active work item found for alert '{ruleName}', ignoring", notification.RuleName); + return; } - string notificationMentions = string.Join(", ", options.NotificationTargets.OrEmpty().Select(target => $"@{target}")); - string investigationLine = string.IsNullOrEmpty(notificationMentions) - ? "Please investigate" - : $"{notificationMentions}, please investigate"; + int workItemId = existingWorkItem.Id; + _logger.LogInformation( + "Found existing work item {workItemId}, resolving alert", + workItemId); + + string currentTags = GetFieldValue(existingWorkItem, "System.Tags") ?? ""; + string newTags = UpdateTags(currentTags, addTag: InactiveAlertTag, removeTag: ActiveAlertTag); - var issue = new NewIssue(issueTitle) + var fields = new Dictionary { - Body = $@":{icon}: Metric state changed to *{notification.State}* + ["System.State"] = "Done", + ["System.Tags"] = newTags, + }; -> {notification.Message?.Replace("\n", "\n> ")} + await client.UpdateWorkItemAsync(options.Project, workItemId, fields, CancellationToken.None); + await client.AddWorkItemCommentAsync( + options.Project, + workItemId, + GenerateComment(notification), + CancellationToken.None); -{metricText} + _logger.LogInformation("Resolved work item {workItemId}", workItemId); + } -{image} + private async Task GetExistingWorkItemAsync(IAzureDevOpsClient client, GrafanaNotification notification) + { + string id = GetUniqueIdentifier(notification); + string automationId = string.Format(BodyLabelTextFormat, id); + AzureDevOpsAlertOptions options = _alertOptions.Value; -[Go to rule]({notification.RuleUrl}) + string wiql = $@"SELECT [System.Id] FROM WorkItems + WHERE [System.TeamProject] = '{options.Project}' + AND [System.AreaPath] UNDER '{options.AreaPath}' + AND [System.Description] CONTAINS '{automationId}' + AND [System.Tags] CONTAINS '{NotificationTag}' + ORDER BY [System.CreatedDate] DESC"; -{investigationLine} + WorkItem[] results = await client.QueryWorkItemsAsync(options.Project, wiql, CancellationToken.None); + return results.FirstOrDefault(); + } -{options.SupplementalBodyText} + internal string GenerateTitle(GrafanaNotification notification) + { + AzureDevOpsAlertOptions options = _alertOptions.Value; + string title = notification.Title; -
-Automation information below, do not change + if (!string.IsNullOrEmpty(options.TitlePrefix)) + { + title = options.TitlePrefix + title; + } -{string.Format(BodyLabelTextFormat, GetUniqueIdentifier(notification))} + return title; + } -
-".Replace("\r\n","\n") - }; + internal string GenerateDescription(GrafanaNotification notification) + { + AzureDevOpsAlertOptions options = _alertOptions.Value; + string metricText = BuildMetricText(notification); + string icon = GetIcon(notification); + string image = !string.IsNullOrEmpty(notification.ImageUrl) + ? $"\"Metric" + : string.Empty; - issue.Labels.Add(NotificationIdLabel); - issue.Labels.Add(ActiveAlertLabel); - foreach (string label in options.AlertLabels.OrEmpty()) - { - issue.Labels.Add(label); - } + string automationId = string.Format(BodyLabelTextFormat, GetUniqueIdentifier(notification)); - foreach (string label in options.EnvironmentLabels.OrEmpty()) - { - issue.Labels.Add(label); - } + return $@"

:{icon}: Metric state changed to {notification.State}

+
{notification.Message?.Replace("\n", "
")}
+{metricText} +{image} +

Go to rule

+

{options.SupplementalBodyText}

+
{automationId}
".Replace("\r\n", "\n"); + } - return issue; + internal string GenerateComment(GrafanaNotification notification) + { + string metricText = BuildMetricText(notification); + string icon = GetIcon(notification); + string image = !string.IsNullOrEmpty(notification.ImageUrl) + ? $"\"Metric" + : string.Empty; + + return $@"

:{icon}: Metric state changed to {notification.State}

+
{notification.Message?.Replace("\n", "
")}
+{metricText} +{image} +

Go to rule

".Replace("\r\n", "\n"); } private static string BuildMetricText(GrafanaNotification notification) @@ -214,154 +252,69 @@ private static string BuildMetricText(GrafanaNotification notification) IEnumerable matches = notification.EvalMatches ?? Enumerable.Empty(); foreach (GrafanaNotificationMatch match in matches) { - metricText.AppendLine($" - *{match.Metric}* {match.Value}"); + metricText.AppendLine($"
  • {match.Metric} {match.Value}
    • "); } - return metricText.ToString(); - } - - private static string GetIcon(GrafanaNotification notification) - { - string icon; - switch (notification.State) + if (metricText.Length > 0) { - case "ok": - icon = "green_heart"; - break; - case "alerting": - icon = "broken_heart"; - break; - case "no_data": - icon = "heavy_multiplication_x"; - break; - case "paused": - icon = "wavy_dash"; - break; - default: - icon = "grey_question"; - break; + return $"
      {metricText}
    "; } - return icon; + return string.Empty; } - private async Task EnsureLabelsAsync(IGitHubClient client, string org, string repo) + private static string GetIcon(GrafanaNotification notification) { - // Assume someone didn't delete the labels, it's an expensive call to make every time - if (s_labelsCreated) + return notification.State switch { - return; - } - - await s_labelLock.WaitAsync(); - try - { - if (s_labelsCreated) - { - return; - } - - var desiredLabels = new[] - { - new NewLabel(NotificationIdLabel, "f957b6"), - new NewLabel(ActiveAlertLabel, "d73a4a"), - new NewLabel(InactiveAlertLabel, "e4e669"), - }; - - await GitHubModifications.CreateLabelsAsync(client, org, repo, _logger, desiredLabels); - - s_labelsCreated = true; - } - finally - { - s_labelLock.Release(); - } + "ok" => "green_heart", + "alerting" => "broken_heart", + "no_data" => "heavy_multiplication_x", + "paused" => "wavy_dash", + _ => "grey_question", + }; } - private async Task CloseExistingNotificationAsync(GrafanaNotification notification) + private static string GetUniqueIdentifier(GrafanaNotification notification) { - string org = _githubOptions.Value.Organization; - string repo = _githubOptions.Value.Repository; - IGitHubClient client = await GetGitHubClientAsync(org, repo); - Issue issue = await GetExistingIssueAsync(client, notification); - if (issue == null) + string id = null; + if (notification.Tags?.TryGetValue(NotificationTagName, out id) ?? false) { - _logger.LogInformation("No active issue found for alert '{ruleName}', ignoring", notification.RuleName); - return; + return id; } - _logger.LogInformation( - "Found existing issue {org}/{repo}#{issueNumber}, replacing {activeTag} with {inactiveTag}", - org, - repo, - issue.Number, - ActiveAlertLabel, - InactiveAlertLabel); - - await GitHubModifications.TryRemoveAsync(() => client.Issue.Labels.RemoveFromIssue(org, repo, issue.Number, ActiveAlertLabel), _logger); - await GitHubModifications.TryCreateAsync(() => - client.Issue.Labels.AddToIssue(org, repo, issue.Number, new[] {InactiveAlertLabel}), - _logger); - - _logger.LogInformation("Adding recurrence comment to {org}/{repo}#{issueNumber}", - org, - repo, - issue.Number); - IssueComment comment = await client.Issue.Comment.Create(org, - repo, - issue.Number, - GenerateNewNotificationComment(notification)); - _logger.LogInformation("Created comment {org}/{repo}#{issue}-issuecomment-{comment}", - org, - repo, - issue.Id, - comment.Id); + return notification.RuleId.ToString(); } - private async Task GetExistingIssueAsync(IGitHubClient client, GrafanaNotification notification) + private static string BuildTagString(params string[] tags) { - string id = GetUniqueIdentifier(notification); - - var searchedLabels = new List - { - NotificationIdLabel - }; - - searchedLabels.AddRange(_githubOptions.Value.EnvironmentLabels.OrEmpty()); - - string automationId = string.Format(BodyLabelTextFormat, id); - var request = new SearchIssuesRequest(automationId) - { - Labels = searchedLabels, - Order = SortDirection.Descending, - SortField = IssueSearchSort.Created, - Type = IssueTypeQualifier.Issue, - In = new[] {IssueInQualifier.Body}, - State = ItemState.Open, - }; - - SearchIssuesResult issues = await client.Search.SearchIssues(request); - - return issues.Items.FirstOrDefault(); + return string.Join("; ", tags); } - private static string GetUniqueIdentifier(GrafanaNotification notification) + private static string UpdateTags(string currentTags, string addTag, string removeTag) { - string id = null; - if (notification.Tags?.TryGetValue(NotificationTagName, out id) ?? false) + var tags = currentTags + .Split(';', StringSplitOptions.RemoveEmptyEntries) + .Select(t => t.Trim()) + .Where(t => !string.Equals(t, removeTag, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (!tags.Contains(addTag, StringComparer.OrdinalIgnoreCase)) { - return id; + tags.Add(addTag); } - return notification.RuleId.ToString(); + return string.Join("; ", tags); } - private async Task GetGitHubClientAsync(string org, string repo) + private static string GetFieldValue(WorkItem workItem, string fieldName) { - return new GitHubClient(_githubClientOptions.Value.ProductHeader) + if (workItem?.Fields != null && workItem.Fields.TryGetValue(fieldName, out object value)) { - Credentials = new Credentials(await _tokenProvider.GetTokenForRepository(org, repo)) - }; + return value?.ToString(); + } + + return null; } private bool IsAuthorized() diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/DotNet.Status.Web.csproj b/src/DotNet.Status.Web/DotNet.Status.Web/DotNet.Status.Web.csproj index 81f26f393..c296f2f6e 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/DotNet.Status.Web.csproj +++ b/src/DotNet.Status.Web/DotNet.Status.Web/DotNet.Status.Web.csproj @@ -8,6 +8,10 @@ OutOfProcess + + + + PreserveNewest diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Options/AzureDevOpsAlertOptions.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Options/AzureDevOpsAlertOptions.cs new file mode 100644 index 000000000..b979684e0 --- /dev/null +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Options/AzureDevOpsAlertOptions.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace DotNet.Status.Web.Options; + +public class AzureDevOpsAlertOptions +{ + public string Organization { get; set; } + public string Project { get; set; } + public string AreaPath { get; set; } + public string WorkItemType { get; set; } = "DNCENG Task"; + public string[] Tags { get; set; } + public string TitlePrefix { get; set; } + public string SupplementalBodyText { get; set; } +} diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs index 9c3fff381..c89df642c 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs @@ -92,6 +92,7 @@ private void ConfigureConfiguration(IServiceCollection services) services.Configure(Configuration.GetSection("IssueMentionForwarding")); services.Configure(Configuration.GetSection("GitHub")); services.Configure(Configuration.GetSection("Grafana")); + services.Configure(Configuration.GetSection("AzureDevOpsAlert")); services.Configure(Configuration.GetSection("Annotations")); services.Configure(Configuration.GetSection("GitHubAppAuth")); services.Configure("dnceng", Configuration.GetSection("AzureDevOps:dnceng")); diff --git a/src/Telemetry/AzureDevOpsClient/AzureDataTypes.cs b/src/Telemetry/AzureDevOpsClient/AzureDataTypes.cs index 864ee03fc..2b307f497 100644 --- a/src/Telemetry/AzureDevOpsClient/AzureDataTypes.cs +++ b/src/Telemetry/AzureDevOpsClient/AzureDataTypes.cs @@ -371,3 +371,19 @@ public sealed class WorkItemRelation public string Rel { get; set; } public string Url { get; set; } } + +public sealed class WiqlQueryResult +{ + public WiqlWorkItemReference[] WorkItems { get; set; } +} + +public sealed class WiqlWorkItemReference +{ + public int Id { get; set; } + public string Url { get; set; } +} + +public sealed class WorkItemQueryResponse +{ + public WorkItem[] Value { get; set; } +} diff --git a/src/Telemetry/AzureDevOpsClient/AzureDevOpsClient.cs b/src/Telemetry/AzureDevOpsClient/AzureDevOpsClient.cs index 6aa1882d3..b81a3c7f4 100644 --- a/src/Telemetry/AzureDevOpsClient/AzureDevOpsClient.cs +++ b/src/Telemetry/AzureDevOpsClient/AzureDevOpsClient.cs @@ -226,6 +226,92 @@ private async Task GetTimelineRaw(string project, int buildId, string id return JsonConvert.DeserializeObject(json); } + public async Task CreateWorkItemAsync(string project, string type, Dictionary fields, string areaPath, CancellationToken cancellationToken = default) + { + StringBuilder builder = GetProjectApiRootBuilder(project); + builder.Append($"wit/workitems/${type}?api-version=7.0"); + + var patchDocuments = new List(); + foreach (var field in fields) + { + patchDocuments.Add(new JsonPatchDocument + { + From = null, + Op = "add", + Path = $"/fields/{field.Key}", + Value = field.Value + }); + } + + patchDocuments.Add(new JsonPatchDocument + { + From = null, + Op = "add", + Path = "/fields/System.AreaPath", + Value = areaPath + }); + + string body = JsonConvert.SerializeObject(patchDocuments); + string json = (await PostJsonResult(builder.ToString(), body, cancellationToken)).Body; + return JsonConvert.DeserializeObject(json); + } + + public async Task UpdateWorkItemAsync(string project, int workItemId, Dictionary fields, CancellationToken cancellationToken = default) + { + StringBuilder builder = GetProjectApiRootBuilder(project); + builder.Append($"wit/workitems/{workItemId}?api-version=7.0"); + + var patchDocuments = new List(); + foreach (var field in fields) + { + patchDocuments.Add(new JsonPatchDocument + { + From = null, + Op = "replace", + Path = $"/fields/{field.Key}", + Value = field.Value + }); + } + + string body = JsonConvert.SerializeObject(patchDocuments); + string json = (await PatchJsonResult(builder.ToString(), body, cancellationToken)).Body; + return JsonConvert.DeserializeObject(json); + } + + public async Task AddWorkItemCommentAsync(string project, int workItemId, string comment, CancellationToken cancellationToken = default) + { + StringBuilder builder = GetProjectApiRootBuilder(project); + builder.Append($"wit/workitems/{workItemId}/comments?api-version=7.0-preview.4"); + + string body = JsonConvert.SerializeObject(new { text = comment }); + await PostJsonResult(builder.ToString(), body, cancellationToken, contentType: "application/json"); + } + + public async Task QueryWorkItemsAsync(string project, string wiql, CancellationToken cancellationToken = default) + { + // Step 1: Run WIQL query to get work item IDs + StringBuilder builder = GetProjectApiRootBuilder(project); + builder.Append("wit/wiql?api-version=7.0"); + + string queryBody = JsonConvert.SerializeObject(new { query = wiql }); + string queryJson = (await PostJsonResult(builder.ToString(), queryBody, cancellationToken, contentType: "application/json")).Body; + var queryResult = JsonConvert.DeserializeObject(queryJson); + + if (queryResult?.WorkItems == null || queryResult.WorkItems.Length == 0) + { + return Array.Empty(); + } + + // Step 2: Fetch full work items + string ids = string.Join(",", queryResult.WorkItems.Select(wi => wi.Id)); + StringBuilder getBuilder = GetProjectApiRootBuilder(project); + getBuilder.Append($"wit/workitems?ids={ids}&api-version=7.0"); + + string itemsJson = (await GetJsonResult(getBuilder.ToString(), cancellationToken)).Body; + var itemsResult = JsonConvert.DeserializeObject(itemsJson); + return itemsResult?.Value ?? Array.Empty(); + } + /// /// The method reads the logs as a stream, line by line and tries to match the regexes in order, one regex per line. /// If the consecutive regexes match the lines, the last match is returned. @@ -427,6 +513,85 @@ private async Task PostJsonResult(string uri, string body, Cancellat } } + private async Task PostJsonResult(string uri, string body, CancellationToken cancellationToken, string contentType) + { + await _parallelism.WaitAsync(cancellationToken); + try + { + await EnsureBearerTokenAsync(cancellationToken); + int retry = 5; + while (true) + { + try + { + var content = new StringContent(body, Encoding.UTF8, contentType); + + using (HttpResponseMessage response = await _httpClient.PostAsync(uri, content, cancellationToken)) + { + response.EnsureSuccessStatusCode(); + string responseBody = await response.Content.ReadAsStringAsync(); + response.Headers.TryGetValues("x-ms-continuationtoken", + out IEnumerable? continuationTokenHeaders); + string? continuationToken = continuationTokenHeaders?.FirstOrDefault(); + return new JsonResult(responseBody, continuationToken); + } + } + catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) + { + throw; + } + catch (Exception) when (retry-- > 0) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + } + } + finally + { + _parallelism.Release(); + } + } + + private async Task PatchJsonResult(string uri, string body, CancellationToken cancellationToken) + { + await _parallelism.WaitAsync(cancellationToken); + try + { + await EnsureBearerTokenAsync(cancellationToken); + int retry = 5; + while (true) + { + try + { + var content = new StringContent(body, Encoding.UTF8, "application/json-patch+json"); + var request = new HttpRequestMessage(new HttpMethod("PATCH"), uri) { Content = content }; + + using (HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken)) + { + response.EnsureSuccessStatusCode(); + string responseBody = await response.Content.ReadAsStringAsync(); + response.Headers.TryGetValues("x-ms-continuationtoken", + out IEnumerable? continuationTokenHeaders); + string? continuationToken = continuationTokenHeaders?.FirstOrDefault(); + return new JsonResult(responseBody, continuationToken); + } + } + catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) + { + throw; + } + catch (Exception) when (retry-- > 0) + { + await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken); + } + } + } + finally + { + _parallelism.Release(); + } + } + public async Task GetChangeDetails(string changeUrl, CancellationToken cancellationToken = default) { var result = await GetJsonResult(changeUrl, cancellationToken); diff --git a/src/Telemetry/AzureDevOpsClient/IAzureDevOpsClient.cs b/src/Telemetry/AzureDevOpsClient/IAzureDevOpsClient.cs index 853d2a75f..621ea0311 100644 --- a/src/Telemetry/AzureDevOpsClient/IAzureDevOpsClient.cs +++ b/src/Telemetry/AzureDevOpsClient/IAzureDevOpsClient.cs @@ -24,6 +24,10 @@ public interface IAzureDevOpsClient public Task GetTimelineAsync(string project, int buildId, string timelineId, CancellationToken cancellationToken); public Task GetChangeDetails(string changeUrl, CancellationToken cancellationToken = default); public Task CreateRcaWorkItem(string project, string title, CancellationToken cancellationToken = default); + public Task CreateWorkItemAsync(string project, string type, Dictionary fields, string areaPath, CancellationToken cancellationToken = default); + public Task UpdateWorkItemAsync(string project, int workItemId, Dictionary fields, CancellationToken cancellationToken = default); + public Task AddWorkItemCommentAsync(string project, int workItemId, string comment, CancellationToken cancellationToken = default); + public Task QueryWorkItemsAsync(string project, string wiql, CancellationToken cancellationToken = default); public Task MatchLogLineSequence(string logUri, IReadOnlyList regexes, CancellationToken cancellationToken = default); public Task GetProjectNameAsync(string id); } diff --git a/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockAzureClient.cs b/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockAzureClient.cs index ba8395f1e..1b9472313 100644 --- a/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockAzureClient.cs +++ b/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockAzureClient.cs @@ -138,4 +138,24 @@ public Task ListBuilds(string project, CancellationToken cancellationTo { throw new NotImplementedException(); } -} + + public Task CreateWorkItemAsync(string project, string type, Dictionary fields, string areaPath, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task UpdateWorkItemAsync(string project, int workItemId, Dictionary fields, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task AddWorkItemCommentAsync(string project, int workItemId, string comment, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task QueryWorkItemsAsync(string project, string wiql, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockTimeoutAzureClient.cs b/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockTimeoutAzureClient.cs index 3e129390d..26a650320 100644 --- a/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockTimeoutAzureClient.cs +++ b/src/Telemetry/Microsoft.DotNet.AzureDevOpsTimelineTests/MockTimeoutAzureClient.cs @@ -83,4 +83,24 @@ public Task ListBuilds(string project, CancellationToken cancellationTo { throw new NotImplementedException(); } + + public Task CreateWorkItemAsync(string project, string type, Dictionary fields, string areaPath, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task UpdateWorkItemAsync(string project, int workItemId, Dictionary fields, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task AddWorkItemCommentAsync(string project, int workItemId, string comment, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public Task QueryWorkItemsAsync(string project, string wiql, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } }