feat: use hash to see if model+cluster prediction is to be reused

Author: abhitrueprogrammer

ConsoleApp2/Analyser.cs

@@ -1,4 +1,5 @@
 using Microsoft.ML.Data;
+using System.Security.Cryptography;
 using System.Text.Json;
 using Microsoft.ML;
 
@@ -25,6 +26,15 @@ internal class Analyser
     {
         private const string FeaturesColumnName = "Features";
 
+        public static string CalculateFileHash(string filePath)
+        {
+            if (!File.Exists(filePath)) return string.Empty;
+            using var sha256 = SHA256.Create();
+            using var stream = File.OpenRead(filePath);
+            var hash = sha256.ComputeHash(stream);
+            return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
+        }
+
         public static IDataView LoadJsonDataForML(MLContext mlContext, string jsonFilePath)
         {
             if (!File.Exists(jsonFilePath))
@@ -125,16 +135,97 @@ public static IEnumerable<IGrouping<uint, CommitPredictionWithData>> GetClusters
             var results = mlContext.Data.CreateEnumerable<CommitPredictionWithData>(predictions, reuseRowObject: false).ToList();
             return results.GroupBy(x => x.PredictedClusterId).OrderBy(g => g.Key);
         }
-        public static IEnumerable<IGrouping<uint, CommitPredictionWithData>> PredictClusterNames(IEnumerable<IGrouping<uint, CommitPredictionWithData>> clusters)
+
+        public static async Task<Dictionary<uint, string>> PredictClusterNamesAsync(IEnumerable<IGrouping<uint, CommitPredictionWithData>> clusters, string labelsFilePath, string modelName = "gemini-2.5-flash")
         {
-            return null;
+            if (File.Exists(labelsFilePath))
+            {
+                var json = File.ReadAllText(labelsFilePath);
+                var dict = JsonSerializer.Deserialize<Dictionary<string, string>>(json);
+                if (dict != null && dict.Count > 0)
+                {
+                    Console.WriteLine($"Loaded cluster names from {labelsFilePath}.");
+                    return dict.ToDictionary(k => uint.Parse(k.Key), v => v.Value);
+                }
+            }
+
+            var apiKey = Environment.GetEnvironmentVariable("GEMINI_API_KEY");
+            if (string.IsNullOrEmpty(apiKey))
+            {
+                Console.WriteLine("GEMINI_API_KEY environment variable not found. Using default cluster numeric names.");
+                return clusters.ToDictionary(g => g.Key, g => $"Cluster {g.Key}");
+            }
+
+            var clusterNames = new System.Collections.Generic.Dictionary<uint, string>();
+            using var httpClient = new System.Net.Http.HttpClient();
+
+            Console.WriteLine("\nPredicting cluster names using Gemini API...");
+            foreach (var cluster in clusters)
+            {
+                var commitsToUse = cluster.Take(10).Select(c => c.CommitName).ToList();
+                var prompt = "Based on the following git commit messages, provide a short 1-3 word category name for this cluster.\n\n" +
+                             "Commits:\n" + string.Join("\n", commitsToUse) + "\n\nCategory name:";
+
+                var requestBody = new
+                {
+                    contents = new[]
+                    {
+                        new
+                        {
+                            parts = new[] { new { text = prompt } }
+                        }
+                    }
+                };
+
+                var url = $"https://generativelanguage.googleapis.com/v1beta/models/{modelName}:generateContent?key={apiKey}";
+                var jsonContent = new System.Net.Http.StringContent(JsonSerializer.Serialize(requestBody), System.Text.Encoding.UTF8, "application/json");
+
+                try
+                {
+                    var response = await httpClient.PostAsync(url, jsonContent);
+                    if (response.IsSuccessStatusCode)
+                    {
+                        var responseString = await response.Content.ReadAsStringAsync();
+                        using var doc = JsonDocument.Parse(responseString);
+                        var text = doc.RootElement
+                            .GetProperty("candidates")[0]
+                            .GetProperty("content")
+                            .GetProperty("parts")[0]
+                            .GetProperty("text").GetString();
+
+                        var cleanedName = text?.Trim().TrimEnd('\r', '\n', '.', '\"', '\'')
+                            .Replace("'", ""); // Additional cleanup for single quotes
+                        clusterNames[cluster.Key] = string.IsNullOrWhiteSpace(cleanedName) ? $"Cluster {cluster.Key}" : cleanedName;
+                        Console.WriteLine($"Cluster {cluster.Key} predicted as: {clusterNames[cluster.Key]}");
+                    }
+                    else
+                    {
+                        Console.WriteLine($"Failed to predict name for Cluster {cluster.Key}. Status: {response.StatusCode}");
+                        clusterNames[cluster.Key] = $"Cluster {cluster.Key}";
+                    }
+                }
+                catch (System.Exception ex)
+                {
+                    Console.WriteLine($"Error predicting name for Cluster {cluster.Key}: {ex.Message}");
+                    clusterNames[cluster.Key] = $"Cluster {cluster.Key}";
+                }
+            }
+
+            // Save for future runs
+            var saveFormat = clusterNames.ToDictionary(k => k.Key.ToString(), v => v.Value);
+            var jsonOut = JsonSerializer.Serialize(saveFormat, new JsonSerializerOptions { WriteIndented = true });
+            File.WriteAllText(labelsFilePath, jsonOut);
+
+            return clusterNames;
         }
-        public static void PrintClusterExamples(IEnumerable<IGrouping<uint, CommitPredictionWithData>> clusters)
+
+        public static void PrintClusterExamples(IEnumerable<IGrouping<uint, CommitPredictionWithData>> clusters, Dictionary<uint, string> clusterNames = null)
         {
             Console.WriteLine("\n--- Cluster Examples ---");
             foreach (var cluster in clusters)
             {
-                Console.WriteLine($"\nCluster {cluster.Key}:");
+                var name = clusterNames != null && clusterNames.TryGetValue(cluster.Key, out var cn) ? cn: $"Cluster {cluster.Key}";
+                Console.WriteLine($"\n{name}:");
                 foreach (var example in cluster.Take(2)) // 2 examples each from each cluster
                 {
                     Console.WriteLine($"  - [{example.Repository}] {example.CommitName}");

ConsoleApp2/Program.cs

@@ -24,25 +24,58 @@ static async Task Main(string[] args)
             // 1. Load data
             var dataView = Analyser.LoadJsonDataForML(mlContext, outputPath);
 
+            var labelsFilePath = "cluster_labels.json";
+
             if (dataView != null)
             {
                 Console.WriteLine($"Loaded {dataView.GetRowCount()} rows into ML.NET IDataView.");
 
+                var hashFilePath = "data_hash.txt";
+                var modelFilePath = "kmeans_model.zip";
+
+                string currentHash = Analyser.CalculateFileHash(outputPath);
+                string savedHash = File.Exists(hashFilePath) ? File.ReadAllText(hashFilePath) : string.Empty;
+
+                if (currentHash != savedHash)
+                {
+                    Console.WriteLine("Data file changed or hash not found. Invalidating cache (K, labels, model)...");
+                    if (File.Exists(kFilePath)) File.Delete(kFilePath);
+                    if (File.Exists(labelsFilePath)) File.Delete(labelsFilePath);
+                    if (File.Exists(modelFilePath)) File.Delete(modelFilePath);
+                    File.WriteAllText(hashFilePath, currentHash);
+                }
+
                 // 2. Split data into 80% train and 20% test
                 var split = Analyser.SplitData(mlContext, dataView);
 
-                // 3. Featurize Text
-                var featurizer = Analyser.FeaturizeText(mlContext);
+                ITransformer model;
+                if (File.Exists(modelFilePath))
+                {
+                    Console.WriteLine($"Loading existing model from {modelFilePath}...");
+                    model = mlContext.Model.Load(modelFilePath, out var schema);
+                }
+                else
+                {
+                    // 3. Featurize Text
+                    var featurizer = Analyser.FeaturizeText(mlContext);
 
-                // 4. Find Best K using Grid Search or load from file
-                int bestK = Analyser.GetOrFindBestK(mlContext, split.TrainSet, featurizer, kFilePath);
+                    // 4. Find Best K using Grid Search or load from file
+                    int bestK = Analyser.GetOrFindBestK(mlContext, split.TrainSet, featurizer, kFilePath);
 
-                // 5. Train KMeans Clusterer
-                var model = Analyser.TrainKMeansClusterer(mlContext, split.TrainSet, featurizer, bestK);
+                    // 5. Train KMeans Clusterer
+                    model = Analyser.TrainKMeansClusterer(mlContext, split.TrainSet, featurizer, bestK);
+
+                    mlContext.Model.Save(model, split.TrainSet.Schema, modelFilePath);
+                    Console.WriteLine($"Saved newly trained model to {modelFilePath}.");
+                }
 
-                // 6. Print 2 examples from each cluster
                 var clusters = Analyser.GetClusters(mlContext, split.TrainSet, model);
-                Analyser.PrintClusterExamples(clusters);
+
+                // 6. Predict cluster names via Gemini (or load from cache)
+                var clusterNames = await Analyser.PredictClusterNamesAsync(clusters, labelsFilePath);
+
+                // 7. Print examples from each cluster with readable names
+                Analyser.PrintClusterExamples(clusters, clusterNames);
             }
         }
     }