Skip to content

Commit 33e66a2

Browse files
CakeRepositoryCakeRepository
authored
Merge pull request #38 from flowdevs-io/dev
Swapped everything from SK to Microsoft.Extensions.AI
2 parents e242141 + fe787a4 commit 33e66a2

26 files changed

Lines changed: 2785 additions & 261 deletions

‎FlowVision/FlowVision.csproj‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
<FileAlignment>512</FileAlignment>
1414
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
1515
<Deterministic>true</Deterministic>
16+
<LangVersion>latest</LangVersion>
1617
<NuGetPackageImportStamp>
1718
</NuGetPackageImportStamp>
1819
</PropertyGroup>
@@ -72,6 +73,9 @@
7273
<Reference Include="Microsoft.Extensions.AI.AzureAIInference, Version=9.4.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
7374
<HintPath>..\packages\Microsoft.Extensions.AI.AzureAIInference.9.4.0-preview.1.25207.5\lib\net462\Microsoft.Extensions.AI.AzureAIInference.dll</HintPath>
7475
</Reference>
76+
<Reference Include="Microsoft.Extensions.AI.OpenAI, Version=9.4.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
77+
<HintPath>..\packages\Microsoft.Extensions.AI.OpenAI.9.4.0-preview.1.25207.5\lib\net462\Microsoft.Extensions.AI.OpenAI.dll</HintPath>
78+
</Reference>
7579
<Reference Include="Microsoft.Extensions.Caching.Abstractions, Version=10.0.0.0, Culture=neutral, PublicKeyToken=adb9793829ddae60, processorArchitecture=MSIL">
7680
<HintPath>..\packages\Microsoft.Extensions.Caching.Abstractions.10.0.0-preview.3.25171.5\lib\net462\Microsoft.Extensions.Caching.Abstractions.dll</HintPath>
7781
</Reference>
@@ -193,6 +197,7 @@
193197
<Compile Include="lib\Classes\MarkdownHelper.cs" />
194198
<Compile Include="lib\Classes\PlaywrightSessionManager.cs" />
195199
<Compile Include="lib\Classes\PluginLogger.cs" />
200+
<Compile Include="lib\Classes\PluginToolExtractor.cs" />
196201
<Compile Include="lib\Classes\SettingsProfileManager.cs" />
197202
<Compile Include="lib\Classes\SpeechRecognitionService.cs" />
198203
<Compile Include="lib\Classes\TaskNotifier.cs" />

‎FlowVision/Form1.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
using System.Threading.Tasks;
99
using System.Windows.Forms;
1010
using FlowVision.lib.Classes;
11-
using Microsoft.SemanticKernel.ChatCompletion;
11+
using Microsoft.Extensions.AI;
1212
// Add System.Speech namespace
1313
using System.Speech.Recognition;
1414
using FlowVision.lib.UI;

‎FlowVision/lib/Classes/PluginToolExtractor.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Reflection;
4+
using Microsoft.Extensions.AI;
5+
6+
namespace FlowVision.lib.Classes
7+
{
8+
/// <summary>
9+
/// Helper class to extract AITools from plugin instances
10+
/// </summary>
11+
public static class PluginToolExtractor
12+
{
13+
/// <summary>
14+
/// Extracts all public instance methods from a plugin and converts them to AITools
15+
/// </summary>
16+
public static List<AITool> ExtractTools(object plugin)
17+
{
18+
var tools = new List<AITool>();
19+
var pluginType = plugin.GetType();
20+
21+
var methods = pluginType.GetMethods(BindingFlags.Public | BindingFlags.Instance);
22+
23+
foreach (var method in methods)
24+
{
25+
// Skip methods from Object base class and special methods (properties, etc.)
26+
if (method.DeclaringType == typeof(object) || method.IsSpecialName)
27+
continue;
28+
29+
// Skip methods declared in parent types (only get methods from the plugin itself)
30+
if (method.DeclaringType != pluginType)
31+
continue;
32+
33+
try
34+
{
35+
var tool = AIFunctionFactory.Create(method, plugin);
36+
tools.Add(tool);
37+
}
38+
catch (Exception)
39+
{
40+
// Skip methods that can't be converted to tools
41+
continue;
42+
}
43+
}
44+
45+
return tools;
46+
}
47+
}
48+
}

‎FlowVision/lib/Classes/ToolDescriptionGenerator.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
using System.Reflection;
55
using System.Text;
66
using FlowVision.lib.Plugins;
7-
using Microsoft.SemanticKernel;
87

98
namespace FlowVision.lib.Classes
109
{

‎FlowVision/lib/Classes/ai/Actioner.cs‎

Lines changed: 50 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,24 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Linq;
4+
using System.Reflection;
45
using System.Text;
56
using System.Threading.Tasks;
67
using System.Windows.Forms;
78
using FlowVision.lib.Plugins;
8-
using Microsoft.Extensions.DependencyInjection;
9-
using Microsoft.SemanticKernel;
10-
using Microsoft.SemanticKernel.ChatCompletion;
11-
using Microsoft.SemanticKernel.Connectors.OpenAI;
9+
using Microsoft.Extensions.AI;
10+
using Azure.AI.OpenAI;
11+
using Azure;
12+
using ChatMessage = Microsoft.Extensions.AI.ChatMessage;
1213

1314
namespace FlowVision.lib.Classes
1415
{
1516
public class Actioner
1617
{
17-
private IChatCompletionService actionerChat;
18-
private ChatHistory actionerHistory;
19-
private Kernel actionerKernel;
18+
private IChatClient actionerChat;
19+
private List<ChatMessage> actionerHistory;
2020
private const string ACTIONER_CONFIG = "actioner";
21-
private const string TOOL_CONFIG = "toolsconfig"; // Added constant for tool config
21+
private const string TOOL_CONFIG = "toolsconfig";
2222

2323
private MultiAgentActioner multiAgentActioner;
2424
private bool useMultiAgentMode = false;
@@ -28,7 +28,7 @@ public class Actioner
2828
// Update the Actioner constructor to support both the delegate and the RichTextBox approaches
2929
public Actioner(Form1.PluginOutputHandler outputHandler)
3030
{
31-
actionerHistory = new ChatHistory();
31+
actionerHistory = new List<ChatMessage>();
3232

3333
// Create a RichTextBox that isn't displayed but used for logging
3434
var hiddenTextBox = new RichTextBox { Visible = false };
@@ -102,10 +102,10 @@ public async Task<string> ExecuteAction(string actionPrompt)
102102
try
103103
{
104104
// Add system message to actioner history
105-
actionerHistory.AddSystemMessage(toolConfig.ActionerSystemPrompt + toolDescriptions);
105+
actionerHistory.Add(new ChatMessage(ChatRole.System, toolConfig.ActionerSystemPrompt + toolDescriptions));
106106

107107
// Add action prompt to actioner history
108-
actionerHistory.AddUserMessage(actionPrompt);
108+
actionerHistory.Add(new ChatMessage(ChatRole.User, actionPrompt));
109109

110110
// Load actioner model config
111111
APIConfig config = APIConfig.LoadConfig(ACTIONER_CONFIG);
@@ -118,93 +118,91 @@ public async Task<string> ExecuteAction(string actionPrompt)
118118
return "Error: Actioner model not configured";
119119
}
120120

121-
// Setup the kernel for actioner with plugins
122-
var builder = Kernel.CreateBuilder();
123-
builder.AddAzureOpenAIChatCompletion(
124-
config.DeploymentName,
125-
config.EndpointURL,
126-
config.APIKey);
121+
// Create Azure OpenAI chat client with IChatClient interface
122+
var azureClient = new AzureOpenAIClient(new Uri(config.EndpointURL), new AzureKeyCredential(config.APIKey));
123+
IChatClient baseChatClient = azureClient.GetChatClient(config.DeploymentName).AsIChatClient();
127124

128-
// Configure OpenAI settings based on toolConfig
129-
var settings = new OpenAIPromptExecutionSettings
130-
{
131-
Temperature = toolConfig.Temperature,
132-
ToolCallBehavior = toolConfig.AutoInvokeKernelFunctions
133-
? Microsoft.SemanticKernel.Connectors.OpenAI.ToolCallBehavior.AutoInvokeKernelFunctions
134-
: Microsoft.SemanticKernel.Connectors.OpenAI.ToolCallBehavior.EnableKernelFunctions
135-
};
125+
// Collect tools based on configuration
126+
var tools = new List<AITool>();
136127

137-
// Add plugins dynamically based on tool configuration
138128
if (toolConfig.EnableCMDPlugin)
139129
{
140-
builder.Plugins.AddFromType<CMDPlugin>();
130+
tools.AddRange(PluginToolExtractor.ExtractTools(new CMDPlugin()));
141131
}
142132

143133
if (toolConfig.EnablePowerShellPlugin)
144134
{
145-
builder.Plugins.AddFromType<PowerShellPlugin>();
135+
tools.AddRange(PluginToolExtractor.ExtractTools(new PowerShellPlugin()));
146136
}
147137

148138
if (toolConfig.EnableScreenCapturePlugin)
149139
{
150-
builder.Plugins.AddFromType<ScreenCaptureOmniParserPlugin>();
140+
tools.AddRange(PluginToolExtractor.ExtractTools(new ScreenCaptureOmniParserPlugin()));
151141
}
152142

153143
if (toolConfig.EnableKeyboardPlugin)
154144
{
155-
builder.Plugins.AddFromType<KeyboardPlugin>();
145+
tools.AddRange(PluginToolExtractor.ExtractTools(new KeyboardPlugin()));
156146
}
157147

158148
if (toolConfig.EnableMousePlugin)
159149
{
160-
builder.Plugins.AddFromType<MousePlugin>();
150+
tools.AddRange(PluginToolExtractor.ExtractTools(new MousePlugin()));
161151
}
162152

163153
if (toolConfig.EnableWindowSelectionPlugin)
164154
{
165-
builder.Plugins.AddFromType<WindowSelectionPlugin>();
155+
tools.AddRange(PluginToolExtractor.ExtractTools(new WindowSelectionPlugin()));
166156
}
167157

168158
if (toolConfig.EnablePlaywrightPlugin)
169159
{
170-
// Expose browser automation utilities including ExecuteScript and CloseBrowser
171-
builder.Plugins.AddFromType<PlaywrightPlugin>();
160+
tools.AddRange(PluginToolExtractor.ExtractTools(new PlaywrightPlugin()));
172161
}
173162

174163
if (toolConfig.EnableRemoteControl)
175164
{
176-
builder.Plugins.AddFromType<RemoteControlPlugin>();
165+
tools.AddRange(PluginToolExtractor.ExtractTools(new RemoteControlPlugin()));
177166
}
178167

179-
actionerKernel = builder.Build();
180-
actionerChat = actionerKernel.GetRequiredService<IChatCompletionService>();
168+
// Configure chat options with tools
169+
var chatOptions = new ChatOptions
170+
{
171+
Temperature = (float)toolConfig.Temperature,
172+
Tools = tools
173+
};
174+
175+
// Build chat client with function invocation if enabled
176+
actionerChat = toolConfig.AutoInvokeKernelFunctions
177+
? new ChatClientBuilder(baseChatClient).UseFunctionInvocation().Build()
178+
: baseChatClient;
181179

182180
// Update loading message to show we're now processing the response
183181
PluginLogger.StopLoadingIndicator();
184182
PluginLogger.StartLoadingIndicator("AI response");
185183

186-
// Process the response
184+
// Process the response with streaming
187185
var responseBuilder = new StringBuilder();
188-
var responseStream = actionerChat.GetStreamingChatMessageContentsAsync(actionerHistory, settings, actionerKernel);
189-
var enumerator = responseStream.GetAsyncEnumerator();
190-
try
186+
await foreach (var update in actionerChat.GetStreamingResponseAsync(actionerHistory, chatOptions))
191187
{
192-
while (await enumerator.MoveNextAsync())
188+
if (update.Text != null)
193189
{
194-
var message = enumerator.Current;
195-
if (message.Content == "None") continue;
196-
responseBuilder.Append(message.Content);
190+
responseBuilder.Append(update.Text);
197191
}
198192
}
199-
finally
200-
{
201-
await enumerator.DisposeAsync();
202-
}
203193

204194
// Task completed successfully
205195
PluginLogger.NotifyTaskComplete("Action Execution", true);
206196

207-
return responseBuilder.ToString();
197+
var response = responseBuilder.ToString();
198+
199+
// Add assistant response to history
200+
if (!string.IsNullOrEmpty(response))
201+
{
202+
actionerHistory.Add(new ChatMessage(ChatRole.Assistant, response));
203+
}
204+
205+
return response;
208206
}
209207
catch (Exception ex)
210208
{
@@ -221,11 +219,11 @@ internal void SetChatHistory(List<LocalChatMessage> chatHistory)
221219
{
222220
if (message.Author == "You")
223221
{
224-
actionerHistory.AddUserMessage(message.Content);
222+
actionerHistory.Add(new ChatMessage(ChatRole.User, message.Content));
225223
}
226224
else if (message.Author == "AI")
227225
{
228-
actionerHistory.AddAssistantMessage(message.Content);
226+
actionerHistory.Add(new ChatMessage(ChatRole.Assistant, message.Content));
229227
}
230228
}
231229

0 commit comments

Comments
 (0)