Add telemetry for AzSdk MCP server tool calls (#12063)

* Add telemetry for AzSdk MCP server tool calls

Author: praveenkuttappan

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Azure.Sdk.Tools.Cli.csproj

@@ -16,6 +16,7 @@
   <ItemGroup>
     <PackageReference Include="Azure.AI.Agents.Persistent" Version="1.0.0" />
     <PackageReference Include="Azure.AI.OpenAI" Version="2.1.0" />
+    <PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.4.0" />
     <PackageReference Include="Microsoft.Extensions.Azure" Version="1.12.0" />
     <PackageReference Include="Microsoft.Graph" Version="5.79.0" />
     <PackageReference Include="OpenTelemetry" Version="1.12.0" />

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Configuration/AzSdkToolsMcpServerConfiguration.cs

@@ -0,0 +1,13 @@
+namespace Azure.Sdk.Tools.Cli.Configuration
+{
+    public class AzSdkToolsMcpServerConfiguration
+    {
+        public const string DefaultName = Constants.TOOLS_ACTIVITY_SOURCE;
+
+        public string Name { get; set; } = DefaultName;
+
+        public string Version { get; set; } = "1.0.0-dev";
+
+        public bool IsTelemetryEnabled { get; set; } = true;
+    }
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Extensions/OpenTelemetryExtensions.cs

@@ -0,0 +1,114 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Reflection;
+using System.Runtime.InteropServices;
+using Azure.Sdk.Tools.Cli.Configuration;
+using Azure.Sdk.Tools.Cli.Telemetry;
+using Azure.Sdk.Tools.Cli.Telemetry.InformationProvider;
+using Azure.Monitor.OpenTelemetry.Exporter;
+using Microsoft.Extensions.Azure;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
+
+namespace Azure.Sdk.Tools.Cli.Extensions;
+
+public static class OpenTelemetryExtensions
+{
+    public static void ConfigureOpenTelemetry(this IServiceCollection services)
+    {
+        services.AddOptions<AzSdkToolsMcpServerConfiguration>()
+            .Configure(options =>
+            {
+                var entryAssembly = Assembly.GetEntryAssembly();
+                var assemblyName = entryAssembly?.GetName() ?? new AssemblyName();
+                if (assemblyName?.Version != null)
+                {
+                    options.Version = assemblyName.Version.ToString();
+                }
+
+                var collectTelemetry = Environment.GetEnvironmentVariable("AZSDKTOOLS_MCP_COLLECT_TELEMETRY");
+                options.IsTelemetryEnabled = string.IsNullOrEmpty(collectTelemetry)
+                    || (bool.TryParse(collectTelemetry, out var shouldCollect) && shouldCollect);
+            });
+
+        services.AddSingleton<ITelemetryService, TelemetryService>();
+
+        if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        {
+            services.AddSingleton<IMachineInformationProvider, WindowsMachineInformationProvider>();
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+        {
+            services.AddSingleton<IMachineInformationProvider, MacOSXMachineInformationProvider>();
+        }
+        else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+        {
+            services.AddSingleton<IMachineInformationProvider, LinuxMachineInformationProvider>();
+        }
+        else
+        {
+            services.AddSingleton<IMachineInformationProvider, DefaultMachineInformationProvider>();
+        }
+
+        EnableAzureMonitor(services);
+    }
+
+    public static void ConfigureOpenTelemetryLogger(this ILoggingBuilder builder)
+    {
+        builder.AddOpenTelemetry(logger =>
+        {
+            logger.AddProcessor(new TelemetryLogRecordEraser());
+        });
+    }
+
+    private static void EnableAzureMonitor(this IServiceCollection services)
+    {
+#if DEBUG
+        services.AddSingleton(sp =>
+        {
+            var forwarder = new AzureEventSourceLogForwarder(sp.GetRequiredService<ILoggerFactory>());
+            forwarder.Start();
+            return forwarder;
+        });
+#endif
+
+        var appInsightsConnectionString = Environment.GetEnvironmentVariable("AZSDKTOOLS_MCP_APPLICATIONINSIGHTS_CONNECTION_STRING");
+        if (string.IsNullOrEmpty(appInsightsConnectionString))
+        {
+            return;
+        }
+
+        services.ConfigureOpenTelemetryTracerProvider((sp, builder) =>
+        {
+            var serverConfig = sp.GetRequiredService<IOptions<AzSdkToolsMcpServerConfiguration>>();
+            if (!serverConfig.Value.IsTelemetryEnabled)
+            {
+                return;
+            }
+
+            builder.AddSource(serverConfig.Value.Name);
+        });
+
+        services.AddOpenTelemetry()
+            .ConfigureResource(r =>
+            {
+                var version = Assembly.GetExecutingAssembly()?.GetName()?.Version?.ToString();
+
+                r.AddService(Constants.TOOLS_ACTIVITY_SOURCE, version)
+                    .AddTelemetrySdk();
+            })
+            .UseAzureMonitorExporter(options =>
+            {
+#if DEBUG
+                options.EnableLiveMetrics = true;
+                options.Diagnostics.IsLoggingEnabled = true;
+                options.Diagnostics.IsLoggingContentEnabled = true;
+#endif
+                options.ConnectionString = appInsightsConnectionString;
+            });
+    }
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs

@@ -6,10 +6,12 @@
 using ModelContextProtocol.Server;
 using Azure.AI.OpenAI;
 using Azure.Sdk.Tools.Cli.Commands;
+using Azure.Sdk.Tools.Cli.Extensions;
 using Azure.Sdk.Tools.Cli.Microagents;
 using Azure.Sdk.Tools.Cli.Helpers;
 using Azure.Sdk.Tools.Cli.Tools;
 using Azure.Sdk.Tools.Cli.Services.ClientUpdate;
+using Azure.Sdk.Tools.Cli.Telemetry;
 
 namespace Azure.Sdk.Tools.Cli.Services
 {
@@ -83,6 +85,10 @@ public static void RegisterCommonServices(IServiceCollection services)
                         return new AzureOpenAIClient(new Uri(ep), credential, options);
                     });
             });
+
+            // Telemetry
+            services.AddSingleton<ITelemetryService, TelemetryService>();
+            services.ConfigureOpenTelemetry();
         }
 
         public static void RegisterInstrumentedMcpTools(IServiceCollection services, string[] args)
@@ -110,7 +116,8 @@ public static void RegisterInstrumentedMcpTools(IServiceCollection services, str
 
                             var loggerFactory = services.GetRequiredService<ILoggerFactory>();
                             var logger = loggerFactory.CreateLogger(toolType);
-                            return new InstrumentedTool(logger, innerTool, toolMethod.Name);
+                            var telemetryService = services.GetRequiredService<ITelemetryService>();
+                            return new InstrumentedTool(telemetryService, logger, innerTool);
                         }));
                     }
                 }

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Telemetry/ITelemetryService.cs

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Diagnostics;
+using ModelContextProtocol.Protocol;
+
+namespace Azure.Sdk.Tools.Cli.Telemetry;
+
+public interface ITelemetryService : IDisposable
+{
+    ValueTask<Activity?> StartActivity(string activityName);
+
+    ValueTask<Activity?> StartActivity(string activityName, Implementation? clientInfo);
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Telemetry/InformationProvider/DefaultMachineInformationProvider.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using Microsoft.Extensions.Logging;
+
+namespace Azure.Sdk.Tools.Cli.Telemetry.InformationProvider;
+
+/// <summary>
+/// Default information provider not tied to any platform specification for DevDeviceId.
+/// </summary>
+internal class DefaultMachineInformationProvider(ILogger<MachineInformationProviderBase> logger)
+    : MachineInformationProviderBase(logger)
+{
+    /// <summary>
+    /// Returns null.
+    /// </summary>
+    /// <returns></returns>
+    public override Task<string?> GetOrCreateDeviceId() => Task.FromResult<string?>(null);
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Telemetry/InformationProvider/IMachineInformationProvider.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace Azure.Sdk.Tools.Cli.Telemetry.InformationProvider;
+
+internal interface IMachineInformationProvider
+{
+    /// <summary>
+    /// Gets existing or creates the device id.  In case the cached id cannot be retrieved, or the
+    /// newly generated id cannot be cached, a value of null is returned.
+    /// </summary>
+    Task<string?> GetOrCreateDeviceId();
+
+    /// <summary>
+    /// Gets a hash of the machine's MAC address.
+    /// </summary>
+    Task<string> GetMacAddressHash();
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Telemetry/InformationProvider/LinuxMachineInformationProvider.cs

@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Runtime.Versioning;
+using Microsoft.Extensions.Logging;
+
+namespace Azure.Sdk.Tools.Cli.Telemetry.InformationProvider;
+
+[SupportedOSPlatform("linux")]
+
+internal class LinuxMachineInformationProvider(ILogger<LinuxMachineInformationProvider> logger) : UnixMachineInformationProvider(logger)
+{
+    private const string PrimaryPathEnvVar = "XDG_CACHE_HOME";
+    private const string SecondaryPathSubDirectory = ".cache";
+
+    /// <summary>
+    /// Gets the base folder for the cache to be stored.
+    /// The final path should be $HOME\.cache\Microsoft\DeveloperTools.
+    /// </summary>
+    public override string GetStoragePath()
+    {
+        var userDir = Environment.GetEnvironmentVariable(PrimaryPathEnvVar);
+
+        // If this comes back as null or empty/whitespace, try environment variable.
+        if (string.IsNullOrWhiteSpace(userDir))
+        {
+            var rootPath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+
+            // If the secondary path is still null/empty/whitespace, then throw as it will lead
+            // to us caching the data in the wrong directory otherwise.
+            if (string.IsNullOrWhiteSpace(rootPath))
+            {
+                throw new InvalidOperationException("linux: Unable to get UserProfile or $HOME folder.");
+            }
+
+            userDir = Path.Combine(rootPath, SecondaryPathSubDirectory);
+        }
+
+        return userDir;
+    }
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Telemetry/InformationProvider/MacOSXMachineInformationProvider.cs

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Runtime.Versioning;
+using Microsoft.Extensions.Logging;
+
+namespace Azure.Sdk.Tools.Cli.Telemetry.InformationProvider;
+
+[SupportedOSPlatform("osx")]
+internal class MacOSXMachineInformationProvider(ILogger<MacOSXMachineInformationProvider> logger)
+    : UnixMachineInformationProvider(logger)
+{
+    /// <summary>
+    /// Retrieves the storage path for application data on macOS.
+    /// </summary>
+    /// <remarks>This method determines the appropriate storage directory by first attempting to retrieve the
+    /// user's profile directory using <see cref="Environment.SpecialFolder.UserProfile"/>. If that is unavailable, it
+    /// falls back to the "HOME"  environment variable. If neither is available, an <see
+    /// cref="InvalidOperationException"/> is thrown.  The returned path is typically located under the
+    /// "Library/Application Support" directory within the user's home folder.</remarks>
+    /// <returns>A string representing the full path to the "Library/Application Support" directory in the user's home folder.
+    /// </returns>
+    /// <exception cref="InvalidOperationException">Thrown if the user's profile directory and the "HOME"
+    /// environment variable are both unavailable or invalid.</exception>
+    public override string GetStoragePath()
+    {
+        var userDir = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+
+        // If this comes back as null or empty/whitespace, try environment variable.
+        if (string.IsNullOrWhiteSpace(userDir))
+        {
+            userDir = Environment.GetEnvironmentVariable("HOME");
+        }
+
+        if (string.IsNullOrWhiteSpace(userDir))
+        {
+            throw new InvalidOperationException("macOS: Unable to get UserProfile or $HOME folder.");
+        }
+
+        var subdirectoryPath = Path.Combine(userDir, "Library", "Application Support");
+        return subdirectoryPath;
+    }
+}

tools/azsdk-cli/Azure.Sdk.Tools.Cli/Telemetry/InformationProvider/MachineInformationProviderBase.cs

@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Net.NetworkInformation;
+using System.Security.Cryptography;
+using System.Text;
+using Microsoft.Extensions.Logging;
+
+namespace Azure.Sdk.Tools.Cli.Telemetry.InformationProvider;
+
+internal abstract class MachineInformationProviderBase(ILogger<MachineInformationProviderBase> logger)
+    : IMachineInformationProvider
+{
+    protected const string MicrosoftDirectory = "Microsoft";
+    protected const string DeveloperToolsDirectory = "DeveloperTools";
+    protected const string NotAvailable = "N/A";
+
+    /// <summary>
+    /// The name of the registry key or file containing device id.
+    /// </summary>
+    protected const string DeviceId = "deviceid";
+
+    private static readonly SHA256 sha256 = SHA256.Create();
+
+    private readonly ILogger<MachineInformationProviderBase> _logger = logger;
+
+    /// <summary>
+    /// <inheritdoc/>
+    /// </summary>
+    public abstract Task<string?> GetOrCreateDeviceId();
+
+    /// <summary>
+    /// <inheritdoc/>
+    /// </summary>
+    public virtual Task<string> GetMacAddressHash()
+    {
+        return Task.Run(() =>
+        {
+            try
+            {
+                var address = GetMacAddress();
+
+                return address != null
+                    ? HashValue(address)
+                    : NotAvailable;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Unable to calculate MAC address hash.");
+                return NotAvailable;
+            }
+        });
+    }
+
+    /// <summary>
+    /// Searches for first network interface card that is up and has a physical address.
+    /// </summary>
+    /// <returns>Hash of the MAC address or <see cref="NotAvailable"/> if none can be found.</returns>
+    protected virtual string? GetMacAddress()
+    {
+        return NetworkInterface.GetAllNetworkInterfaces()
+            .Where(x => x.OperationalStatus == OperationalStatus.Up && x.NetworkInterfaceType != NetworkInterfaceType.Loopback)
+            .Select(x => x.GetPhysicalAddress().ToString())
+            .FirstOrDefault(x => !string.IsNullOrEmpty(x));
+    }
+
+    /// <summary>
+    /// Generates a new device identifier.  Conditions that have to be met are:
+    /// <list type="bullet">
+    /// <item>Randomly generated UUID/GUID</item>
+    /// <item>Follows the 8-4-4-4-12 format (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)</item>
+    /// <item>Contains only lowercase characters and hyphens</item>
+    /// </list>
+    /// </summary>
+    /// <returns>The generated device identifier.</returns>
+    protected string GenerateDeviceId() => Guid.NewGuid().ToString("D").ToLowerInvariant();
+
+    /// <summary>
+    /// Generates a SHA-256 of the given value.
+    /// </summary>
+    protected string HashValue(string value)
+    {
+        var hashInput = sha256.ComputeHash(Encoding.UTF8.GetBytes(value));
+        return BitConverter.ToString(hashInput).Replace("-", string.Empty).ToLowerInvariant();
+    }
+}