Merge pull request #49 from flowdevs-io/test

Add Gemini and local plugin support; UI and config updates

Author: CakeRepository

FlowVision/AIProviderConfigForm.cs

@@ -211,6 +211,7 @@
     </Compile>
     <Compile Include="lib\Classes\AgentCoordinator.cs" />
     <Compile Include="lib\Classes\ai\AgentRole.cs" />
+    <Compile Include="lib\Classes\ai\AIClientFactory.cs" />
     <Compile Include="lib\Classes\ai\LMStudioActioner.cs" />
     <Compile Include="lib\Classes\ai\MultiAgentActioner.cs" />
     <Compile Include="lib\Classes\LMStudioConfig.cs" />
@@ -250,7 +251,9 @@
     <Compile Include="lib\Classes\UI\ActivityMonitor.cs">
       <SubType>UserControl</SubType>
     </Compile>
+    <Compile Include="lib\Plugins\ClipboardPlugin.cs" />
     <Compile Include="lib\Plugins\CMDPlugin.cs" />
+    <Compile Include="lib\Plugins\FileSystemPlugin.cs" />
     <Compile Include="lib\Plugins\KeyboardPlugin.cs" />
     <Compile Include="lib\Plugins\MousePlugin.cs" />
     <Compile Include="lib\Plugins\PowershellPlugin.cs" />
@@ -301,6 +304,7 @@
   </ItemGroup>
   <ItemGroup>
     <Content Include="recursive-control-icon.ico" />
+    <EmbeddedResource Include="Models\icon_detect.onnx" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <Import Project="..\packages\System.ValueTuple.4.6.1\build\net471\System.ValueTuple.targets" Condition="Exists('..\packages\System.ValueTuple.4.6.1\build\net471\System.ValueTuple.targets')" />
@@ -327,13 +331,6 @@
     </ItemGroup>
     <Copy SourceFiles="@(TesseractNative)" DestinationFolder="$(OutputPath)" SkipUnchangedFiles="true" />
   </Target>
-  <!-- Copy OmniParser model to output directory -->
-  <Target Name="CopyOmniParserModel" AfterTargets="AfterBuild">
-    <ItemGroup>
-      <OmniParserModel Include="Models\icon_detect.onnx" />
-    </ItemGroup>
-    <Copy SourceFiles="@(OmniParserModel)" DestinationFolder="$(OutputPath)\models" SkipUnchangedFiles="true" />
-  </Target>
   <Import Project="..\packages\CefSharp.Common.135.0.170\build\CefSharp.Common.targets" Condition="Exists('..\packages\CefSharp.Common.135.0.170\build\CefSharp.Common.targets')" />
   <Import Project="..\packages\Tesseract.5.2.0\build\Tesseract.targets" Condition="Exists('..\packages\Tesseract.5.2.0\build\Tesseract.targets')" />
   <Import Project="..\packages\Fody.6.9.2\build\Fody.targets" Condition="Exists('..\packages\Fody.6.9.2\build\Fody.targets')" />

FlowVision/FlowVision.csproj

@@ -1,47 +1,19 @@
 using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Data;
-using System.Drawing;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using System.Windows.Forms;
 using FlowVision.lib.Classes;
 
 namespace FlowVision
 {
     public partial class OmniParserForm : Form
     {
-        private OmniParserConfig _config;
-
         public OmniParserForm()
         {
             InitializeComponent();
-            _config = OmniParserConfig.LoadConfig();
-        }
-
-        private void saveButton_Click(object sender, EventArgs e)
-        {
-            string url = omniParserServerURL.Text.Trim();
-            if (string.IsNullOrEmpty(url))
-            {
-                MessageBox.Show("Please enter a valid URL.");
-                return;
-            }
-
-            // Save the URL to the config file
-            _config.ServerURL = url;
-            _config.SaveConfig();
-
-            // Optionally, you can close the form after saving
-            this.Close();
         }
 
         private void OmniParserForm_Load(object sender, EventArgs e)
         {
-            // Load the URL from the config file
-            omniParserServerURL.Text = _config.ServerURL;
+            // Form is now just an informational status display
         }
     }
 }

FlowVision/OmniParserForm.Designer.cs

@@ -14,6 +14,20 @@ static class Program
         [STAThread]
         static void Main()
         {
+            // Preload the ONNX model on a background thread so it's ready when needed
+            Task.Run(() =>
+            {
+                try
+                {
+                    // Accessing the Instance property triggers the model loading
+                    var parser = FlowVision.lib.Classes.SimpleOmniParser.Instance;
+                }
+                catch
+                {
+                    // Ignore startup errors - they will be caught/logged when the user actually tries to use it
+                }
+            });
+
             Application.EnableVisualStyles();
             Application.SetCompatibleTextRenderingDefault(false);
             Application.Run(new Form1());

FlowVision/OmniParserForm.cs

@@ -4,93 +4,76 @@
 
 namespace FlowVision.lib.Classes
 {
-    /// <summary>
-    /// Configuration for LM Studio local AI integration
-    /// </summary>
     public class LMStudioConfig
     {
-        /// <summary>
-        /// LM Studio server endpoint (default: http://localhost:1234/v1)
-        /// </summary>
         public string EndpointURL { get; set; } = "http://localhost:1234/v1";
-        
-        /// <summary>
-        /// Model name (e.g., "gpt-3.5-turbo", "local-model", or whatever LM Studio shows)
-        /// Can be left as default - LM Studio typically uses the loaded model automatically
-        /// </summary>
         public string ModelName { get; set; } = "local-model";
-        
-        /// <summary>
-        /// API key (LM Studio doesn't require one, but field kept for compatibility)
-        /// Use "lm-studio" or "not-needed" as placeholder
-        /// </summary>
-        public string APIKey { get; set; } = "lm-studio";
-        
-        /// <summary>
-        /// Whether to use LM Studio or fall back to Azure
-        /// </summary>
-        public bool Enabled { get; set; } = false;
-        
-        /// <summary>
-        /// Temperature setting for local model
-        /// </summary>
         public double Temperature { get; set; } = 0.7;
-        
-        /// <summary>
-        /// Max tokens for completion
-        /// </summary>
         public int MaxTokens { get; set; } = 2048;
-        
-        /// <summary>
-        /// Timeout in seconds for LM Studio requests
-        /// </summary>
-        public int TimeoutSeconds { get; set; } = 300;
+        public bool Enabled { get; set; } = false;
+        public string APIKey { get; set; } = "lm-studio"; // OpenAI client requires a key even if local
+        public int TimeoutSeconds { get; set; } = 120;
 
-        private static string ConfigFilePath()
-        {
-            return Path.Combine(
-                Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
-                "FlowVision",
-                "lmstudioconfig.json");
-        }
+        // Track if the config was successfully loaded from disk
+        [System.Text.Json.Serialization.JsonIgnore]
+        public bool IsValid { get; set; } = true;
+
+        private static string ConfigFilePath => Path.Combine(
+            Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
+            "FlowVision",
+            "lmstudioconfig.json");
 
         public static LMStudioConfig LoadConfig()
         {
             try
             {
-                Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath()));
+                Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath));
 
-                if (File.Exists(ConfigFilePath()))
+                if (File.Exists(ConfigFilePath))
                 {
-                    string jsonContent = File.ReadAllText(ConfigFilePath());
-                    if (!string.IsNullOrWhiteSpace(jsonContent))
+                    string jsonContent = File.ReadAllText(ConfigFilePath);
+                    // Basic validation for empty file
+                    if (string.IsNullOrWhiteSpace(jsonContent))
                     {
-                        var config = JsonSerializer.Deserialize<LMStudioConfig>(jsonContent);
-                        return config ?? new LMStudioConfig();
+                        return new LMStudioConfig { IsValid = false };
+                    }
+
+                    var config = JsonSerializer.Deserialize<LMStudioConfig>(jsonContent);
+                    if (config != null)
+                    {
+                        config.IsValid = true;
+                        return config;
                     }
                 }
             }
             catch (Exception ex)
             {
-                Console.WriteLine($"Error loading LM Studio config: {ex.Message}");
+                // Return a config marked as invalid so the UI can warn the user
+                return new LMStudioConfig 
+                {
+                    Enabled = false,
+                    IsValid = false
+                };
             }
 
+            // Return default if file doesn't exist
             return new LMStudioConfig();
         }
 
         public void SaveConfig()
         {
             try
             {
-                Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath()));
+                Directory.CreateDirectory(Path.GetDirectoryName(ConfigFilePath));
 
                 var options = new JsonSerializerOptions { WriteIndented = true };
                 string jsonContent = JsonSerializer.Serialize(this, options);
-                File.WriteAllText(ConfigFilePath(), jsonContent);
+                File.WriteAllText(ConfigFilePath, jsonContent);
             }
             catch (Exception ex)
             {
                 Console.WriteLine($"Error saving LM Studio config: {ex.Message}");
+                throw; // Re-throw so the UI can show the error
             }
         }
     }

FlowVision/Program.cs

@@ -82,6 +82,54 @@ private static void Initialize()
             }
         }
 
+        /// <summary>
+        /// Search for specific text in an image and return its bounding box.
+        /// Returns the first match found.
+        /// </summary>
+        public static async Task<Rectangle?> FindTextLocationAsync(Bitmap image, string searchText)
+        {
+            if (!_isAvailable || _engine == null || string.IsNullOrWhiteSpace(searchText))
+                return null;
+
+            return await Task.Run(() =>
+            {
+                try
+                {
+                    lock (_lock)
+                    {
+                        using (var pix = PixConverter.ToPix(image))
+                        using (var page = _engine.Process(pix))
+                        using (var iter = page.GetIterator())
+                        {
+                            iter.Begin();
+                            do
+                            {
+                                // Get text at current iterator level (Word)
+                                string currentText = iter.GetText(PageIteratorLevel.Word)?.Trim();
+                                
+                                // Simple case-insensitive match
+                                if (!string.IsNullOrWhiteSpace(currentText) && 
+                                    currentText.Equals(searchText, StringComparison.OrdinalIgnoreCase))
+                                {
+                                    // Found exact match! Get bounding box.
+                                    if (iter.TryGetBoundingBox(PageIteratorLevel.Word, out var rect))
+                                    {
+                                        return (Rectangle?)new Rectangle(rect.X1, rect.Y1, rect.Width, rect.Height);
+                                    }
+                                }
+                            } while (iter.Next(PageIteratorLevel.Word));
+                        }
+                    }
+                    return (Rectangle?)null;
+                }
+                catch (Exception ex)
+                {
+                    PluginLogger.LogError("OcrHelper", "FindTextLocationAsync", $"OCR search failed: {ex.Message}");
+                    return (Rectangle?)null;
+                }
+            });
+        }
+
         /// <summary>
         /// Extract text from a bitmap image
         /// </summary>