Merge pull request #1354 from yileicn/master

yileicn · web-flow · commit b52cbf57f05a · 2026-05-22T17:36:08.000+08:00
add Conversation Log Cleanup
diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs
@@ -23,6 +23,8 @@ public class CleanConversationSetting
     public int BatchSize { get; set; }
     public int MessageLimit { get; set; }
     public int BufferHours { get; set; }
+    public int LogRetentionDays { get; set; }
+    public int LogBatchSize { get; set; } = 2000;
     public IEnumerable<string> ExcludeAgentIds { get; set; } = new List<string>();
 }
 
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
@@ -198,6 +198,11 @@ Task<DateTimePagination<ConversationStateLogModel>> GetConversationStateLogs(str
         => throw new NotImplementedException();
     #endregion
 
+    #region Log Cleanup
+    Task<int> DeleteOldConversationLogs(int retentionDays, int batchSize)
+        => throw new NotImplementedException();
+    #endregion
+
     #region Instruction Log
     Task<bool> SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
         => throw new NotImplementedException();
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs
@@ -178,6 +178,15 @@ public async Task<DateTimePagination<ConversationStateLogModel>> GetConversation
     }
     #endregion
 
+    #region Log Cleanup
+    public async Task<int> DeleteOldConversationLogs(int retentionDays, int batchSize = 2000)
+    {
+        // For file repository, we might not need to implement this fully or we can just return 0
+        // as it's typically used for local dev. Implementing it would require iterating all conversations.
+        return await Task.FromResult(0);
+    }
+    #endregion
+
     #region Instruction Log
     public async Task<bool> SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
     {
diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs
diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs b/src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs
@@ -1,7 +1,8 @@
 using BotSharp.Abstraction.Messaging.JsonConverters;
 using BotSharp.Core.MCP.Settings;
 using BotSharp.Core.Users.Services;
-using BotSharp.OpenAPI.BackgroundServices;
+using BotSharp.OpenAPI.Hooks;
+using BotSharp.OpenAPI.RuleTriggers;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.Cookies;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
@@ -14,6 +15,10 @@
 using Microsoft.Net.Http.Headers;
 using Microsoft.OpenApi;
 using System.Text.Json.Serialization;
+using BotSharp.Abstraction.Crontab;
+using BotSharp.Abstraction.Rules;
+
+
 #if NET8_0
 using Microsoft.OpenApi.Models;
 #endif
@@ -36,7 +41,8 @@ public static IServiceCollection AddBotSharpOpenAPI(this IServiceCollection serv
         bool enableValidation)
     {
         services.AddScoped<IUserIdentity, UserIdentity>();
-        services.AddHostedService<ConversationTimeoutService>();
+        
+        services.AddCrontabServices();
 
         // Add bearer authentication
         var schema = "MIXED_SCHEME";
@@ -223,6 +229,18 @@ private static async Task OnTicketReceivedContext(TicketReceivedContext context)
         }
     }
 
+    public static IServiceCollection AddCrontabServices(this IServiceCollection services)
+    {
+        services.AddScoped<ICrontabHook, IdleConversationCleanupCrontabHook>();
+        services.AddScoped<ICrontabSource, IdleConversationCleanupRuleTrigger>();
+        services.AddScoped<IRuleTrigger, IdleConversationCleanupRuleTrigger>();
+
+        services.AddScoped<ICrontabHook, ConversationLogCleanupCrontabHook>();
+        services.AddScoped<ICrontabSource, ConversationLogCleanupRuleTrigger>();
+        services.AddScoped<IRuleTrigger, ConversationLogCleanupRuleTrigger>();
+        return services;
+    }
+
     /// <summary>
     /// Use Swagger/OpenAPI
     /// </summary>
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Hooks/ConversationLogCleanupCrontabHook.cs b/src/Infrastructure/BotSharp.OpenAPI/Hooks/ConversationLogCleanupCrontabHook.cs
@@ -0,0 +1,69 @@
+using BotSharp.Abstraction.Conversations.Settings;
+using BotSharp.Abstraction.Crontab;
+using BotSharp.Abstraction.Crontab.Models;
+using BotSharp.Abstraction.Repositories;
+using BotSharp.OpenAPI.RuleTriggers;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Threading.Tasks;
+
+namespace BotSharp.OpenAPI.Hooks
+{
+    public class ConversationLogCleanupCrontabHook : ICrontabHook
+    {
+        private readonly ConversationSetting _settings;
+        private readonly IBotSharpRepository _db;
+        private readonly ILogger<ConversationLogCleanupCrontabHook> _logger;
+        private readonly IHostApplicationLifetime _appLifetime;
+        
+        public string[]? Triggers => new[] { nameof(ConversationLogCleanupRuleTrigger) };
+
+        public ConversationLogCleanupCrontabHook(
+            ConversationSetting settings,
+            IBotSharpRepository db,
+            ILogger<ConversationLogCleanupCrontabHook> logger,
+            IHostApplicationLifetime appLifetime)
+        {
+            _settings = settings;
+            _db = db;
+            _logger = logger;
+            _appLifetime = appLifetime;
+        }
+
+        public async Task OnCronTriggered(CrontabItem item)
+        {
+            var cleanSetting = _settings.CleanSetting;
+
+            if (cleanSetting == null || !cleanSetting.Enable || cleanSetting.LogRetentionDays <= 0) return;
+
+            int totalDeleted = 0;
+            var cancellationToken = _appLifetime.ApplicationStopping;
+            
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                var deletedCount = await _db.DeleteOldConversationLogs(cleanSetting.LogRetentionDays, cleanSetting.LogBatchSize);
+                if (deletedCount == 0) break;
+                
+                totalDeleted += deletedCount;
+                _logger.LogInformation($"Cleaned {deletedCount} conversation logs older than {cleanSetting.LogRetentionDays} days in this batch.");
+                
+                try
+                {
+                    // Sleep slightly to yield database resources, will throw TaskCanceledException on shutdown
+                    await Task.Delay(1000, cancellationToken);
+                }
+                catch (OperationCanceledException)
+                {
+                    _logger.LogWarning("Conversation log cleanup was interrupted due to application shutdown.");
+                    break;
+                }
+            }
+
+            if (totalDeleted > 0)
+            {
+                _logger.LogInformation($"Successfully cleaned a total of {totalDeleted} conversation logs older than {cleanSetting.LogRetentionDays} days.");
+            }
+        }
+    }
+}
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Hooks/IdleConversationCleanupCrontabHook.cs b/src/Infrastructure/BotSharp.OpenAPI/Hooks/IdleConversationCleanupCrontabHook.cs
@@ -0,0 +1,53 @@
+using BotSharp.Abstraction.Conversations;
+using BotSharp.Abstraction.Conversations.Settings;
+using BotSharp.Abstraction.Crontab;
+using BotSharp.Abstraction.Crontab.Models;
+using BotSharp.OpenAPI.RuleTriggers;
+using Microsoft.Extensions.Logging;
+
+namespace BotSharp.OpenAPI.Hooks
+{
+    public class IdleConversationCleanupCrontabHook : ICrontabHook
+    {
+        private readonly ConversationSetting _settings;
+        private readonly IConversationService _conversationService;
+        private readonly ILogger<IdleConversationCleanupCrontabHook> _logger;
+
+        public string[]? Triggers => new[] { nameof(IdleConversationCleanupRuleTrigger) };
+
+        public IdleConversationCleanupCrontabHook(
+            ConversationSetting settings,
+            IConversationService conversationService,
+            ILogger<IdleConversationCleanupCrontabHook> logger)
+        {
+            _settings = settings;
+            _conversationService = conversationService;
+            _logger = logger;
+        }
+
+        public async Task OnCronTriggered(CrontabItem item)
+        {
+            var cleanSetting = _settings.CleanSetting;
+
+            if (cleanSetting == null || !cleanSetting.Enable) return;
+
+            try
+            {
+                var batchSize = cleanSetting.BatchSize;
+                var limit = cleanSetting.MessageLimit;
+                var bufferHours = cleanSetting.BufferHours;
+                var excludeAgentIds = cleanSetting.ExcludeAgentIds ?? new List<string>();
+                var conversationIds = await _conversationService.GetIdleConversations(batchSize, limit, bufferHours, excludeAgentIds);
+
+                if (!conversationIds.IsNullOrEmpty())
+                {
+                    await _conversationService.DeleteConversations(conversationIds);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error occurred closing conversations.");
+            }
+        }
+    }
+}
diff --git a/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/ConversationLogCleanupRuleTrigger.cs b/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/ConversationLogCleanupRuleTrigger.cs
@@ -0,0 +1,26 @@
+using BotSharp.Abstraction.Conversations.Enums;
+using BotSharp.Abstraction.Crontab;
+using BotSharp.Abstraction.Crontab.Models;
+using BotSharp.Abstraction.Rules;
+
+namespace BotSharp.OpenAPI.RuleTriggers
+{
+    public class ConversationLogCleanupRuleTrigger : IRuleTrigger, ICrontabSource
+    {
+        public string Channel => ConversationChannel.Crontab;
+        public string Name => nameof(ConversationLogCleanupRuleTrigger);
+        public string EntityType { get; set; } = string.Empty;
+        public string EntityId { get; set; } = string.Empty;
+
+        public CrontabItem GetCrontabItem()
+        {
+            return new CrontabItem
+            {
+                Title = nameof(ConversationLogCleanupRuleTrigger),
+                Description = "Clean up old conversation logs daily",
+                Cron = "0 6 * * *", // Run at 6:00 AM UTC (Midnight Chicago Standard Time)
+                TriggerType = CronTabItemTriggerType.MessageQueue
+            };
+        }
+    }
+}
diff --git a/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/IdleConversationCleanupRuleTrigger.cs b/src/Infrastructure/BotSharp.OpenAPI/RuleTriggers/IdleConversationCleanupRuleTrigger.cs
@@ -0,0 +1,26 @@
+using BotSharp.Abstraction.Conversations.Enums;
+using BotSharp.Abstraction.Crontab;
+using BotSharp.Abstraction.Crontab.Models;
+using BotSharp.Abstraction.Rules;
+
+namespace BotSharp.OpenAPI.RuleTriggers
+{
+    public class IdleConversationCleanupRuleTrigger : IRuleTrigger, ICrontabSource
+    {
+        public string Channel => ConversationChannel.Crontab;
+        public string Name => nameof(IdleConversationCleanupRuleTrigger);
+        public string EntityType { get; set; } = string.Empty;
+        public string EntityId { get; set; } = string.Empty;
+
+        public CrontabItem GetCrontabItem()
+        {
+            return new CrontabItem
+            {
+                Title = nameof(IdleConversationCleanupRuleTrigger),
+                Description = "Clean up idle conversations hourly",
+                Cron = "0 * * * *",
+                TriggerType = CronTabItemTriggerType.MessageQueue
+            };
+        }
+    }
+}
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs
@@ -139,6 +139,37 @@ public async Task<DateTimePagination<ConversationStateLogModel>> GetConversation
     }
     #endregion
 
+    #region Log Cleanup
+    public async Task<int> DeleteOldConversationLogs(int retentionDays, int batchSize)
+    {
+        if (retentionDays <= 0) return 0;
+        var threshold = DateTime.UtcNow.AddDays(-retentionDays);
+
+        var contentLogFilter = Builders<ConversationContentLogDocument>.Filter.Lt(x => x.CreatedTime, threshold);
+        var stateLogFilter = Builders<ConversationStateLogDocument>.Filter.Lt(x => x.CreatedTime, threshold);
+
+        var contentDocsToDelete = await _dc.ContentLogs.Find(contentLogFilter).Limit(batchSize).Project(x => x.Id).ToListAsync();
+        long contentDeletedCount = 0;
+        if (contentDocsToDelete.Any())
+        {
+            var deleteFilter = Builders<ConversationContentLogDocument>.Filter.In(x => x.Id, contentDocsToDelete);
+            var contentDeleted = await _dc.ContentLogs.DeleteManyAsync(deleteFilter);
+            contentDeletedCount = contentDeleted.DeletedCount;
+        }
+
+        var stateDocsToDelete = await _dc.StateLogs.Find(stateLogFilter).Limit(batchSize).Project(x => x.Id).ToListAsync();
+        long stateDeletedCount = 0;
+        if (stateDocsToDelete.Any())
+        {
+            var deleteFilter = Builders<ConversationStateLogDocument>.Filter.In(x => x.Id, stateDocsToDelete);
+            var stateDeleted = await _dc.StateLogs.DeleteManyAsync(deleteFilter);
+            stateDeletedCount = stateDeleted.DeletedCount;
+        }
+
+        return (int)(contentDeletedCount + stateDeletedCount);
+    }
+    #endregion
+
     #region Instruction Log
     public async Task<bool> SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
     {
diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json
@@ -707,6 +707,8 @@
       "BatchSize": 50,
       "MessageLimit": 2,
       "BufferHours": 12,
+      "LogRetentionDays": 30,
+      "LogBatchSize": 2000,
       "ExcludeAgentIds": []
     },
     "RateLimit": {
diff --git a/tests/BotSharp.LLM.Tests/appsettings.json b/tests/BotSharp.LLM.Tests/appsettings.json
@@ -201,6 +201,8 @@
       "BatchSize": 50,
       "MessageLimit": 2,
       "BufferHours": 12,
+      "LogRetentionDays": 30,
+      "LogBatchSize": 2000,
       "ExcludeAgentIds": []
     },
     "RateLimit": {

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,8 @@ public class CleanConversationSetting`
`23`	`23`	`public int BatchSize { get; set; }`
`24`	`24`	`public int MessageLimit { get; set; }`
`25`	`25`	`public int BufferHours { get; set; }`
	`26`	`+ public int LogRetentionDays { get; set; }`
	`27`	`+ public int LogBatchSize { get; set; } = 2000;`
`26`	`28`	`public IEnumerable<string> ExcludeAgentIds { get; set; } = new List<string>();`
`27`	`29`	`}`
`28`	`30`