diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Contract/MCPTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Contract/MCPTool.cs index c45e58dfe9b..bb5ed85c959 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Contract/MCPTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Contract/MCPTool.cs @@ -3,32 +3,31 @@ using System.CommandLine; using System.CommandLine.Invocation; -namespace Azure.Sdk.Tools.Cli.Contract +namespace Azure.Sdk.Tools.Cli.Contract; + +/// +/// This is the base class defining how an MCP enabled tool will interface with the server. +/// +/// This covers: +/// - route registration/disambiguation +/// - compilation trim avoidance for reflection-included MCP tools +/// +public abstract class MCPTool { - /// - /// This is the base class defining how an MCP enabled tool will interface with the server. - /// - /// This covers: - /// - route registration/disambiguation - /// - compilation trim avoidance for reflection-included MCP tools - /// - public abstract class MCPTool - { - public MCPTool() { } + public MCPTool() { } - public Command? Command; + public Command? Command; - public int ExitCode { get; set; } = 0; + public int ExitCode { get; set; } = 0; - public void SetFailure(int exitCode = 1) - { - ExitCode = exitCode; - } + public void SetFailure(int exitCode = 1) + { + ExitCode = exitCode; + } - public CommandGroup[] CommandHierarchy { get; set; } = []; + public CommandGroup[] CommandHierarchy { get; set; } = []; - public abstract Command GetCommand(); + public abstract Command GetCommand(); - public abstract Task HandleCommand(InvocationContext ctx, CancellationToken ct); - } + public abstract Task HandleCommand(InvocationContext ctx, CancellationToken ct); } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Microagents/MicroagentHostServiceTests.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Microagents/MicroagentHostServiceTests.cs index e885271d872..c8baee0ff44 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Microagents/MicroagentHostServiceTests.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Microagents/MicroagentHostServiceTests.cs @@ -3,6 +3,8 @@ using Azure.AI.OpenAI; using Azure.Sdk.Tools.Cli.Microagents; using Azure.Sdk.Tools.Cli.Microagents.Tools; +using Azure.Sdk.Tools.Cli.Helpers; +using NUnit.Framework; using Moq; using OpenAI.Chat; @@ -23,7 +25,8 @@ public void Setup() chatClientMock = new Mock(); openAIClientMock.Setup(client => client.GetChatClient(It.IsAny())) .Returns(chatClientMock.Object); - microagentHostService = new MicroagentHostService(openAIClientMock.Object, loggerMock.Object); + var tokenUsageHelper = new TokenUsageHelper(Mock.Of()); + microagentHostService = new MicroagentHostService(openAIClientMock.Object, loggerMock.Object, tokenUsageHelper); } [Test] diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/ExampleToolTests.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/ExampleToolTests.cs index 90f71bc959d..678777061a6 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/ExampleToolTests.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Tests/Tools/ExampleToolTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using Moq; +using NUnit.Framework; using NUnit.Framework.Internal; using Azure.Core; using Azure.Sdk.Tools.Cli.Helpers; @@ -22,6 +23,7 @@ internal class ExampleToolTests private MockGitHubService? mockGitHubService; private Mock? mockProcessHelper; private Mock? mockPowershellHelper; + private Mock? mockMicroagentHostService; [SetUp] public void Setup() @@ -33,6 +35,7 @@ public void Setup() mockGitHubService = new MockGitHubService(); mockProcessHelper = new Mock(); mockPowershellHelper = new Mock(); + mockMicroagentHostService = new Mock(); // Set up Azure service mock to return a mock credential var mockCredential = new Mock(); @@ -58,6 +61,8 @@ public void Setup() mockGitHubService, mockProcessHelper.Object, mockPowershellHelper.Object, + tokenUsageHelper: new TokenUsageHelper(mockOutput.Object), + mockMicroagentHostService.Object, #pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. null #pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. @@ -153,7 +158,7 @@ public void GetCommand_ReturnsCommandWithCorrectSubCommands() Assert.That(command.Name, Is.EqualTo("demo")); Assert.That(command.Description, Does.Contain("Comprehensive demonstration")); - Assert.That(command.Subcommands.Count, Is.EqualTo(7)); + Assert.That(command.Subcommands.Count, Is.EqualTo(8)); var subCommandNames = command.Subcommands.Select(sc => sc.Name).ToList(); Assert.That(subCommandNames, Does.Contain("azure")); @@ -163,6 +168,7 @@ public void GetCommand_ReturnsCommandWithCorrectSubCommands() Assert.That(subCommandNames, Does.Contain("error")); Assert.That(subCommandNames, Does.Contain("process")); Assert.That(subCommandNames, Does.Contain("powershell")); + Assert.That(subCommandNames, Does.Contain("microagent")); } [Test] @@ -236,7 +242,6 @@ public async Task DemonstratePowershellExecution_Success() { mockPowershellHelper!.Setup(p => p.Run(It.IsAny(), It.IsAny())) .ReturnsAsync(new ProcessResult { ExitCode = 0, }); - var result = await tool.DemonstratePowershellExecution("foobar"); Assert.That(result.ResponseError, Is.Null); @@ -245,4 +250,16 @@ public async Task DemonstratePowershellExecution_Success() Assert.That(result.Result, Is.Empty); Assert.That(result.Details?["exit_code"], Is.EqualTo("0")); } + + [Test] + public async Task DemonstrateMicroagentFibonacci_Success() + { + mockMicroagentHostService!.Setup(m => m.RunAgentToCompletion(It.IsAny>(), It.IsAny())) + .ReturnsAsync(13); + + var response = await tool.DemonstrateMicroagentFibonacci(7); + + Assert.That(response.ResponseError, Is.Null); + Assert.That(response.Result as string, Does.Contain("Fibonacci(7) = 13")); + } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs index f9e45ff1cd9..c401f8ce86a 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Helpers/TokenUsageHelper.cs @@ -1,80 +1,24 @@ namespace Azure.Sdk.Tools.Cli.Helpers; -public class TokenUsageHelper +public class TokenUsageHelper(IOutputHelper outputHelper) { - protected double PromptTokens { get; set; } - protected double CompletionTokens { get; set; } - protected double InputCost { get; set; } - protected double OutputCost { get; set; } - protected double TotalCost { get; set; } - public List Models { get; set; } = []; + protected double PromptTokens { get; set; } = 0; + protected double CompletionTokens { get; set; } = 0; + protected IEnumerable ModelsUsed { get; set; } = []; - public TokenUsageHelper(string model, long inputTokens, long outputTokens) + public void Add(string model, long inputTokens, long outputTokens) { - PromptTokens = inputTokens; - CompletionTokens = outputTokens; - Models = [model]; - SetCost(model); + ModelsUsed = ModelsUsed.Union([model]); + PromptTokens += inputTokens; + CompletionTokens += outputTokens; } - protected TokenUsageHelper() { } - - private void SetCost(string model) + public void LogUsage() { - var oneMillion = 1000000; - double inputPrice, outputPrice; - - // Prices assume the slightly more expensive regional model pricing - if (model == "gpt-4o") - { - (inputPrice, outputPrice) = (2.75, 11); - } - else if (model == "gpt-4o-mini") - { - (inputPrice, outputPrice) = (0.165, 0.66); - } - if (model == "gpt-4.1") - { - (inputPrice, outputPrice) = (2, 8); - } - else if (model == "gpt-4.1-mini") - { - (inputPrice, outputPrice) = (0.4, 1.60); - } - else if (model == "o3-mini") - { - (inputPrice, outputPrice) = (1.21, 4.84); - } - else - { - return; - } - - - InputCost = PromptTokens / oneMillion * inputPrice; - OutputCost = CompletionTokens / oneMillion * outputPrice; - } + var models = string.Join(", ", ModelsUsed); - public void LogCost() - { - var _inputCost = InputCost == 0 ? "?" : InputCost.ToString("F3"); - var _outputCost = OutputCost == 0 ? "?" : OutputCost.ToString("F3"); - var _totalCost = (InputCost + OutputCost) == 0 ? "?" : (InputCost + OutputCost).ToString("F3"); - var models = string.Join(", ", Models); - Console.WriteLine("--------------------------------------------------------------------------------"); - Console.WriteLine($"[{models}] Usage (cost / tokens):"); - Console.WriteLine($" Input: ${_inputCost} / {PromptTokens}"); - Console.WriteLine($" Output: ${_outputCost} / {CompletionTokens}"); - Console.WriteLine($" Total: ${_totalCost} / {PromptTokens + CompletionTokens}"); - Console.WriteLine("--------------------------------------------------------------------------------"); + outputHelper.OutputConsole("--------------------------------------------------------------------------------"); + outputHelper.OutputConsole($"[token usage][{models}] input: {PromptTokens}, output: {CompletionTokens}, total: {PromptTokens + CompletionTokens}"); + outputHelper.OutputConsole("--------------------------------------------------------------------------------"); } - - public static TokenUsageHelper operator +(TokenUsageHelper a, TokenUsageHelper? b) => new() - { - Models = a.Models.Union(b?.Models ?? []).ToList(), - PromptTokens = a.PromptTokens + (b?.PromptTokens ?? 0), - CompletionTokens = a.CompletionTokens + (b?.CompletionTokens ?? 0), - InputCost = a.InputCost + (b?.InputCost ?? 0), - OutputCost = a.OutputCost + (b?.OutputCost ?? 0), - }; -} \ No newline at end of file +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Microagents/MicroagentHostService.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Microagents/MicroagentHostService.cs index 48ac2aff16e..344b9556681 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Microagents/MicroagentHostService.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Microagents/MicroagentHostService.cs @@ -1,10 +1,11 @@ using System.ComponentModel; using Azure.AI.OpenAI; +using Azure.Sdk.Tools.Cli.Helpers; using OpenAI.Chat; namespace Azure.Sdk.Tools.Cli.Microagents; -public class MicroagentHostService(AzureOpenAIClient openAI, ILogger logger) : IMicroagentHostService +public class MicroagentHostService(AzureOpenAIClient openAI, ILogger logger, TokenUsageHelper tokenUsageHelper) : IMicroagentHostService { private const string ExitToolName = "Exit"; @@ -60,6 +61,10 @@ public async Task RunAgentToCompletion(Microagent age // Request the chat completion logger.LogDebug("Sending conversation history with {MessageCount} messages to model '{Model}'", conversationHistory.Count, agentDefinition.Model); var response = await chatClient.CompleteChatAsync(conversationHistory, chatCompletionOptions, ct); + if (null != response.Value.Usage) + { + tokenUsageHelper.Add(agentDefinition.Model, response.Value.Usage.InputTokenCount, response.Value.Usage.OutputTokenCount); + } var toolCall = response.Value.ToolCalls.Single(); logger.LogInformation("Model called tool '{ToolName}'", toolCall.FunctionName); @@ -104,7 +109,7 @@ public async Task RunAgentToCompletion(Microagent age conversationHistory.Add(ChatMessage.CreateToolMessage(toolCall.Id, toolResult)); } - throw new Exception("Agent did not return a result within the maximum number of iterations"); + throw new Exception($"Agent did not return a result within the maximum number of {agentDefinition.MaxToolCalls} iterations"); } /// diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs index c003b91faf9..d5645cd4c07 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Program.cs @@ -46,7 +46,8 @@ public static WebApplicationBuilder CreateAppBuilder(string[] args) WebApplicationBuilder builder = WebApplication.CreateBuilder(args); builder.Services.AddOpenTelemetry() - .WithTracing(b => { + .WithTracing(b => + { b.AddSource(Constants.TOOLS_ACTIVITY_SOURCE) .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() @@ -55,24 +56,21 @@ public static WebApplicationBuilder CreateAppBuilder(string[] args) }) .UseOtlpExporter(); - // Log everything to stderr in mcp mode so the client doesn't try to interpret stdout messages that aren't json rpc - var logErrorThreshold = isCLI ? LogLevel.Error : LogLevel.Debug; - builder.Logging.AddConsole(consoleLogOptions => { + // Log everything to stderr in mcp mode so the client doesn't try to interpret stdout messages that aren't json rpc + var logErrorThreshold = isCLI ? LogLevel.Error : LogLevel.Debug; consoleLogOptions.LogToStandardErrorThreshold = logErrorThreshold; }); - // Skip verbose azure client logging + // Skip azure client logging noise builder.Logging.AddFilter((category, level) => { - var isAzureClient = category!.StartsWith("Azure.", StringComparison.Ordinal); - var isToolsClient = category!.StartsWith("Azure.Sdk.Tools.", StringComparison.Ordinal); - if (isAzureClient && !isToolsClient) - { - return level >= LogLevel.Warning; - } - return level >= logErrorThreshold; + if (debug || null == category) { return level >= logLevel; } + var isAzureClient = category.StartsWith("Azure.", StringComparison.Ordinal); + var isToolsClient = category.StartsWith("Azure.Sdk.Tools.", StringComparison.Ordinal); + if (isAzureClient && !isToolsClient) { return level >= LogLevel.Error; } + return level >= logLevel; }); // add the console logger diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentService.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentService.cs index 353d168960b..5f00ec72ed2 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentService.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentService.cs @@ -8,14 +8,20 @@ public interface IAzureAgentService string ProjectEndpoint { get; } Task DeleteAgents(CancellationToken ct); - Task<(string, TokenUsageHelper)> QueryFiles(List files, string session, string query, CancellationToken ct); + Task QueryFiles(List files, string session, string query, CancellationToken ct); } -public class AzureAgentService(IAzureService azureService, ILogger logger, string? _projectEndpoint, string? _model) : IAzureAgentService +public class AzureAgentService( + IAzureService azureService, + ILogger logger, + TokenUsageHelper tokenUsageHelper, + string? _projectEndpoint, + string? _model +) : IAzureAgentService { public string ProjectEndpoint { get; } = _projectEndpoint ?? defaultProjectEndpoint; - private static readonly string defaultProjectEndpoint = "https://azsdk-engsys-ai.services.ai.azure.com/api/projects/azsdk-engsys-ai"; - private readonly string model = _model ?? "gpt-4.1-mini"; + private static readonly string defaultProjectEndpoint = "https://azsdk-engsys-openai.services.ai.azure.com/api/projects/azsdk-engsys-openai-foundry"; + private readonly string model = _model ?? "o3-mini"; private readonly PersistentAgentsClient client = new(_projectEndpoint ?? defaultProjectEndpoint, azureService.GetCredential()); @@ -38,37 +44,37 @@ Find other log lines in addition to the final error that may be descriptive of t public async Task DeleteAgents(CancellationToken ct = default) { - logger.LogInformation("Deleting agents in project '{ProjectEndpoint}'", ProjectEndpoint); + logger.LogDebug("Deleting agents in project '{ProjectEndpoint}'", ProjectEndpoint); AsyncPageable agents = client.Administration.GetAgentsAsync(cancellationToken: ct); await foreach (var agent in agents) { - logger.LogInformation("Deleting agent {AgentId} ({AgentName})", agent.Id, agent.Name); + logger.LogDebug("Deleting agent {AgentId} ({AgentName})", agent.Id, agent.Name); await client.Administration.DeleteAgentAsync(agent.Id, ct); } AsyncPageable threads = client.Threads.GetThreadsAsync(cancellationToken: ct); await foreach (var thread in threads) { - logger.LogInformation("Deleting thread {ThreadId}", thread.Id); + logger.LogDebug("Deleting thread {ThreadId}", thread.Id); await client.Threads.DeleteThreadAsync(thread.Id, ct); } AsyncPageable vectorStores = client.VectorStores.GetVectorStoresAsync(cancellationToken: ct); await foreach (var vectorStore in vectorStores) { - logger.LogInformation("Deleting vector store {VectorStoreId} ({VectorStoreName})", vectorStore.Id, vectorStore.Name); + logger.LogDebug("Deleting vector store {VectorStoreId} ({VectorStoreName})", vectorStore.Id, vectorStore.Name); await client.VectorStores.DeleteVectorStoreAsync(vectorStore.Id, ct); } var files = await client.Files.GetFilesAsync(cancellationToken: ct); foreach (var file in files.Value) { - logger.LogInformation("Deleting file {FileId} ({FileName})", file.Id, file.Filename); + logger.LogDebug("Deleting file {FileId} ({FileName})", file.Id, file.Filename); await client.Files.DeleteFileAsync(file.Id, ct); } } - public async Task<(string, TokenUsageHelper)> QueryFiles(List files, string session, string query, CancellationToken ct) + public async Task QueryFiles(List files, string session, string query, CancellationToken ct) { List uploaded = []; foreach (var file in files) @@ -135,10 +141,12 @@ public async Task DeleteAgents(CancellationToken ct = default) } } + tokenUsageHelper.Add(model, run.Usage.PromptTokens, run.Usage.CompletionTokens); + // NOTE: in the future we will want to keep these around if the user wants to keep querying the file in a session - logger.LogDebug("Deleting temporary resources: agent {AgentId}, vector store {VectorStoreId}, {fileCount} files", agent.Id, vectorStore.Id, uploaded.Count); + logger.LogInformation("Deleting temporary resources: agent {AgentId}, vector store {VectorStoreId}, {fileCount} files", agent.Id, vectorStore.Id, uploaded.Count); + await DeleteAgents(ct); - var tokenUsage = new TokenUsageHelper(model, run.Usage.PromptTokens, run.Usage.CompletionTokens); - return (string.Join("\n", response), tokenUsage); + return string.Join("\n", response); } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentServiceFactory.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentServiceFactory.cs index 4634c2f2bbc..9be650699c2 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentServiceFactory.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/Azure/Agent/AzureAgentServiceFactory.cs @@ -1,3 +1,5 @@ +using Azure.Sdk.Tools.Cli.Helpers; + namespace Azure.Sdk.Tools.Cli.Services; public interface IAzureAgentServiceFactory @@ -5,10 +7,10 @@ public interface IAzureAgentServiceFactory IAzureAgentService Create(string? projectEndpoint = null, string? model = null); } -public class AzureAgentServiceFactory(IAzureService azureService, ILogger logger) : IAzureAgentServiceFactory +public class AzureAgentServiceFactory(IAzureService azureService, ILogger logger, TokenUsageHelper tokenUsageHelper) : IAzureAgentServiceFactory { public IAzureAgentService Create(string? projectEndpoint = null, string? model = null) { - return new AzureAgentService(azureService, logger, projectEndpoint, model); + return new AzureAgentService(azureService, logger, tokenUsageHelper, projectEndpoint, model); } } diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs index 46e65977012..f246aeda30d 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Services/ServiceRegistrations.cs @@ -27,10 +27,10 @@ public static void RegisterCommonServices(IServiceCollection services) { // Services services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddScoped(); // Language Check Services (Composition-based) services.AddSingleton(); @@ -58,6 +58,8 @@ public static void RegisterCommonServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + // Add as scoped so we can track/update usage across tools and services per request for logging/telemetry + services.AddScoped(); // Process Helper Classes services.AddSingleton(); @@ -107,7 +109,7 @@ public static void RegisterInstrumentedMcpTools(IServiceCollection services, str { if (toolMethod.GetCustomAttribute() is not null) { - services.AddSingleton((Func)(services => + services.AddScoped((Func)(services => { var options = new McpServerToolCreateOptions { Services = services, SerializerOptions = serializerOptions }; var innerTool = toolMethod.IsStatic diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Example/ExampleTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Example/ExampleTool.cs index 3979df2b716..804af78a4e6 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Example/ExampleTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Example/ExampleTool.cs @@ -11,6 +11,7 @@ using Azure.Sdk.Tools.Cli.Helpers; using ModelContextProtocol.Server; using OpenAI.Chat; +using Azure.Sdk.Tools.Cli.Microagents; namespace Azure.Sdk.Tools.Cli.Tools.Example; @@ -26,6 +27,7 @@ public class ExampleTool : MCPTool private const string ErrorSubCommand = "error"; private const string ProcessSubCommand = "process"; private const string PowershellSubCommand = "powershell"; + private const string MicroagentSubCommand = "microagent"; // Dependencies injected via constructor private readonly ILogger logger; @@ -36,6 +38,8 @@ public class ExampleTool : MCPTool private readonly AzureOpenAIClient openAIClient; private readonly IProcessHelper processHelper; private readonly IPowershellHelper powershellHelper; + private readonly TokenUsageHelper tokenUsageHelper; + private readonly IMicroagentHostService microagentHostService; // CLI Options and Arguments private readonly Argument aiInputArg = new( @@ -67,6 +71,11 @@ public class ExampleTool : MCPTool description: "Message to pass to the PowerShell script via parameter") { Arity = ArgumentArity.ExactlyOne }; + private readonly Option fibonacciIndexOption = new( + name: "--fibonacci", + description: "Index (0-based) of Fibonacci number to compute using micro-agent") + { IsRequired = true }; + private readonly Option tenantOption = new(["--tenant", "-t"], "Tenant ID"); private readonly Option languageOption = new(["--language", "-l"], "Programming language of the repository"); private readonly Option promptOption = new(["--prompt", "-p"], "AI prompt text"); @@ -81,6 +90,8 @@ public ExampleTool( IGitHubService gitHubService, IProcessHelper processHelper, IPowershellHelper powershellHelper, + TokenUsageHelper tokenUsageHelper, + IMicroagentHostService microagentHostService, AzureOpenAIClient openAIClient ) : base() { @@ -92,6 +103,8 @@ AzureOpenAIClient openAIClient this.openAIClient = openAIClient; this.processHelper = processHelper; this.powershellHelper = powershellHelper; + this.tokenUsageHelper = tokenUsageHelper; + this.microagentHostService = microagentHostService; // Set command hierarchy - results in: azsdk example CommandHierarchy = [ @@ -139,6 +152,11 @@ public override Command GetCommand() powershellCmd.AddArgument(powershellMessageArg); powershellCmd.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); + // Microagent Fibonacci demo sub-command + var microagentCmd = new Command(MicroagentSubCommand, "Demonstrate micro-agent looping tool calls to compute Fibonacci"); + microagentCmd.AddOption(fibonacciIndexOption); + microagentCmd.SetHandler(async ctx => { await HandleCommand(ctx, ctx.GetCancellationToken()); }); + parentCommand.Add(azureCmd); parentCommand.Add(devopsCmd); parentCommand.Add(githubCmd); @@ -146,6 +164,7 @@ public override Command GetCommand() parentCommand.Add(errorCmd); parentCommand.Add(processCmd); parentCommand.Add(powershellCmd); + parentCommand.Add(microagentCmd); return parentCommand; } @@ -163,6 +182,7 @@ public override async Task HandleCommand(InvocationContext ctx, CancellationToke ErrorSubCommand => await DemonstrateErrorHandling(ctx.ParseResult.GetValueForArgument(errorInputArg), ctx.ParseResult.GetValueForOption(forceFailureOption), ct), ProcessSubCommand => await DemonstrateProcessExecution(ctx.ParseResult.GetValueForArgument(processSleepArg), ct), PowershellSubCommand => await DemonstratePowershellExecution(ctx.ParseResult.GetValueForArgument(powershellMessageArg), ct), + MicroagentSubCommand => await DemonstrateMicroagentFibonacci(ctx.ParseResult.GetValueForOption(fibonacciIndexOption), ct), _ => new ExampleServiceResponse { ResponseError = $"Unknown command: {commandName}" } }; @@ -294,6 +314,8 @@ public async Task DemonstrateAIService(string userPrompt }; var response = await chatClient.CompleteChatAsync(messages, cancellationToken: ct); + tokenUsageHelper.Add(model, response.Value.Usage.InputTokenCount, response.Value.Usage.OutputTokenCount); + tokenUsageHelper.LogUsage(); return new ExampleServiceResponse { @@ -481,5 +503,64 @@ public async Task DemonstratePowershellExecution(string } } } + + public record Fibonacci + { + public int Index { get; set; } + public int Previous { get; set; } + public int Current { get; set; } + } + + [McpServerTool(Name = "azsdk_example_microagent_fibonacci"), Description("Demonstrates micro-agent computing Nth Fibonacci number via iterative tool calls")] + public async Task DemonstrateMicroagentFibonacci(int n, CancellationToken ct = default) + { + try + { + if (n < 2) + { + SetFailure(); + return new DefaultCommandResponse { ResponseError = "--fibonacci must be >= 2 to run the micro-agent" }; + } + + var advanceTool = AgentTool.FromFunc( + name: "advance_state", + description: "Advances state by one step", + invokeHandler: (input, ct) => + { + return Task.FromResult(new Fibonacci + { + Index = input.Index + 1, + Previous = input.Current, + Current = input.Previous + input.Current + }); + }); + + // Avoid mentioning 'fibonacci' in the instructions so the LLM doesn't try to calculate it directly + var instructions = $""" + Call advance_state repeatedly until the returned index == {n}. + Return the '{nameof(Fibonacci.Current)}' value when index == {n}. + Initial state is {nameof(Fibonacci.Index)}=1, {nameof(Fibonacci.Previous)}=0, {nameof(Fibonacci.Current)}=1. + """; + + var agent = new Microagent + { + Instructions = instructions, + MaxToolCalls = 7, + Tools = [advanceTool], + }; + + var resultValue = await microagentHostService.RunAgentToCompletion(agent, ct); + + tokenUsageHelper.LogUsage(); + return new DefaultCommandResponse { Result = $"Fibonacci({n}) = {resultValue}" }; + } + catch (Exception ex) + { + tokenUsageHelper.LogUsage(); + logger.LogError(ex, "Error demonstrating micro-agent Fibonacci for n={n}", n); + SetFailure(); + return new DefaultCommandResponse { ResponseError = $"Failed to compute Fibonacci({n}): {ex.Message}" }; + } + } } #endif diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Pipeline/PipelineAnalysisTool.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Pipeline/PipelineAnalysisTool.cs index 9ed93c83b8c..b044e0ff5c0 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Pipeline/PipelineAnalysisTool.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/Pipeline/PipelineAnalysisTool.cs @@ -28,31 +28,31 @@ public class PipelineAnalysisTool : MCPTool private readonly IDevOpsService devopsService; private readonly IAzureAgentServiceFactory azureAgentServiceFactory; private readonly ILogAnalysisHelper logAnalysisHelper; - private ITestHelper testHelper; - private readonly IOutputHelper output; + private readonly ITestHelper testHelper; private readonly ILogger logger; + private readonly IOutputHelper output; + private readonly TokenUsageHelper tokenUsageHelper; private IAzureAgentService azureAgentService; - private TokenUsageHelper usage; private bool initialized = false; private readonly HttpClient httpClient = new(); - private BuildHttpClient _buildClient; + private BuildHttpClient buildClientValue; private BuildHttpClient buildClient { get { Initialize(); - return _buildClient; + return buildClientValue; } } - private TestResultsHttpClient _testClient; + private TestResultsHttpClient testClientValue; private TestResultsHttpClient testClient { get { Initialize(); - return _testClient; + return testClientValue; } } @@ -70,8 +70,9 @@ public PipelineAnalysisTool( IAzureAgentServiceFactory azureAgentServiceFactory, ILogAnalysisHelper logAnalysisHelper, ITestHelper testHelper, + ILogger logger, IOutputHelper output, - ILogger logger + TokenUsageHelper tokenUsageHelper ) : base() { this.azureService = azureService; @@ -79,8 +80,9 @@ ILogger logger this.azureAgentServiceFactory = azureAgentServiceFactory; this.logAnalysisHelper = logAnalysisHelper; this.testHelper = testHelper; - this.output = output; this.logger = logger; + this.output = output; + this.tokenUsageHelper = tokenUsageHelper; CommandHierarchy = [ @@ -116,14 +118,14 @@ public override async Task HandleCommand(InvocationContext ctx, CancellationToke { var result = await AnalyzePipelineFailureLogs(project ?? projectFromLink, buildId, [logId], analyzeWithAgent, ct); ctx.ExitCode = ExitCode; - usage?.LogCost(); + tokenUsageHelper.LogUsage(); output.Output(result); } else { var result = await AnalyzePipeline(project ?? projectFromLink, buildId, analyzeWithAgent, ct); ctx.ExitCode = ExitCode; - usage?.LogCost(); + tokenUsageHelper.LogUsage(); output.Output(result); } } @@ -174,14 +176,14 @@ private void Initialize(bool auth = true) var token = azureService.GetCredential(Constants.MICROSOFT_CORP_TENANT).GetToken(new TokenRequestContext(tokenScope), CancellationToken.None); var tokenCredential = new VssOAuthAccessTokenCredential(token.Token); var connection = new VssConnection(new Uri(Constants.AZURE_SDK_DEVOPS_BASE_URL), tokenCredential); - _buildClient = connection.GetClient(); - _testClient = connection.GetClient(); + buildClientValue = connection.GetClient(); + testClientValue = connection.GetClient(); } else { var connection = new VssConnection(new Uri(Constants.AZURE_SDK_DEVOPS_BASE_URL), null); - _buildClient = connection.GetClient(); - _testClient = connection.GetClient(); + buildClientValue = connection.GetClient(); + testClientValue = connection.GetClient(); } initialized = true; @@ -400,7 +402,7 @@ public async Task AnalyzePipelineFailureLogs(string? projec return response; } - var (result, _usage) = await azureAgentService.QueryFiles(logs, session, "Why did this pipeline fail?", ct); + var result = await azureAgentService.QueryFiles(logs, session, "Why did this pipeline fail?", ct); // Sometimes chat gpt likes to wrap the json in markdown if (result.StartsWith("```json")) { @@ -416,9 +418,6 @@ public async Task AnalyzePipelineFailureLogs(string? projec File.Delete(log); } - - usage = usage != null ? usage + _usage : _usage; - try { return JsonSerializer.Deserialize(result);