add Conversation Log Cleanup

Author: yileicn

src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs

@@ -23,6 +23,8 @@ public class CleanConversationSetting
     public int BatchSize { get; set; }
     public int MessageLimit { get; set; }
     public int BufferHours { get; set; }
+    public int LogRetentionDays { get; set; }
+    public int LogBatchSize { get; set; } = 2000;
     public IEnumerable<string> ExcludeAgentIds { get; set; } = new List<string>();
 }
 

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -198,6 +198,11 @@ Task<DateTimePagination<ConversationStateLogModel>> GetConversationStateLogs(str
         => throw new NotImplementedException();
     #endregion
 
+    #region Log Cleanup
+    Task<int> DeleteOldConversationLogs(int retentionDays, int batchSize)
+        => throw new NotImplementedException();
+    #endregion
+
     #region Instruction Log
     Task<bool> SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
         => throw new NotImplementedException();

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs

@@ -178,6 +178,15 @@ public async Task<DateTimePagination<ConversationStateLogModel>> GetConversation
     }
     #endregion
 
+    #region Log Cleanup
+    public async Task<int> DeleteOldConversationLogs(int retentionDays, int batchSize = 2000)
+    {
+        // For file repository, we might not need to implement this fully or we can just return 0
+        // as it's typically used for local dev. Implementing it would require iterating all conversations.
+        return await Task.FromResult(0);
+    }
+    #endregion
+
     #region Instruction Log
     public async Task<bool> SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
     {

src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationLogCleanupService.cs

@@ -0,0 +1,103 @@
+using BotSharp.Abstraction.Conversations.Settings;
+using BotSharp.Abstraction.Repositories;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BotSharp.OpenAPI.BackgroundServices
+{
+    public class ConversationLogCleanupService : BackgroundService
+    {
+        private readonly IServiceProvider _services;
+        private readonly ILogger<ConversationLogCleanupService> _logger;
+
+        public ConversationLogCleanupService(IServiceProvider services, ILogger<ConversationLogCleanupService> logger)
+        {
+            _services = services;
+            _logger = logger;
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            _logger.LogInformation("Conversation Log Cleanup Service is running...");
+
+            _ = Task.Run(async () =>
+            {
+                await DoWork(stoppingToken);
+            });
+        }
+
+        private async Task DoWork(CancellationToken stoppingToken)
+        {
+            _logger.LogInformation("Conversation Log Cleanup Service is doing work...");
+
+            try
+            {
+                while (true)
+                {
+                    stoppingToken.ThrowIfCancellationRequested();
+                    // Run once a day
+                    var delay = Task.Delay(TimeSpan.FromHours(24), stoppingToken);
+                    try
+                    {
+                        await CleanOldLogsAsync();
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, $"Error occurred cleaning old conversation logs.");
+                    }
+                    await delay;
+                }
+            }
+            catch (OperationCanceledException) { }
+        }
+
+        public override async Task StopAsync(CancellationToken stoppingToken)
+        {
+            _logger.LogInformation("Conversation Log Cleanup Service is stopping.");
+            await base.StopAsync(stoppingToken);
+        }
+
+        private async Task CleanOldLogsAsync()
+        {
+            using var scope = _services.CreateScope();
+            var settings = scope.ServiceProvider.GetRequiredService<ConversationSetting>();
+            var cleanSetting = settings.CleanSetting;
+
+            if (cleanSetting == null || !cleanSetting.Enable || cleanSetting.LogRetentionDays <= 0) return;
+
+            var locker = scope.ServiceProvider.GetRequiredService<BotSharp.Abstraction.Infrastructures.IDistributedLocker>();
+            var lockResource = nameof(ConversationLogCleanupService);
+            await locker.LockAsync(lockResource, async () =>
+            {
+                await ExecuteCleanup(scope.ServiceProvider, cleanSetting.LogRetentionDays, cleanSetting.LogBatchSize);
+            }, timeout: 3600); // 1 hour lock timeout to prevent other pods from running it
+        }
+
+        private async Task ExecuteCleanup(IServiceProvider serviceProvider, int retentionDays, int batchSize)
+        {
+            var db = serviceProvider.GetRequiredService<IBotSharpRepository>();
+            int totalDeleted = 0;
+            
+            while (true)
+            {
+                var deletedCount = await db.DeleteOldConversationLogs(retentionDays, batchSize);
+                if (deletedCount == 0) break;
+                
+                totalDeleted += deletedCount;
+                _logger.LogInformation($"Cleaned {deletedCount} conversation logs older than {retentionDays} days in this batch.");
+                
+                // Sleep slightly to yield database resources
+                await Task.Delay(1000);
+            }
+
+            if (totalDeleted > 0)
+            {
+                _logger.LogInformation($"Successfully cleaned a total of {totalDeleted} conversation logs older than {retentionDays} days.");
+            }
+        }
+    }
+}

src/Infrastructure/BotSharp.OpenAPI/BotSharpOpenApiExtensions.cs

@@ -37,6 +37,7 @@ public static IServiceCollection AddBotSharpOpenAPI(this IServiceCollection serv
     {
         services.AddScoped<IUserIdentity, UserIdentity>();
         services.AddHostedService<ConversationTimeoutService>();
+        services.AddHostedService<ConversationLogCleanupService>();
 
         // Add bearer authentication
         var schema = "MIXED_SCHEME";

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Log.cs

@@ -139,6 +139,37 @@ public async Task<DateTimePagination<ConversationStateLogModel>> GetConversation
     }
     #endregion
 
+    #region Log Cleanup
+    public async Task<int> DeleteOldConversationLogs(int retentionDays, int batchSize)
+    {
+        if (retentionDays <= 0) return 0;
+        var threshold = DateTime.UtcNow.AddDays(-retentionDays);
+
+        var contentLogFilter = Builders<ConversationContentLogDocument>.Filter.Lt(x => x.CreatedTime, threshold);
+        var stateLogFilter = Builders<ConversationStateLogDocument>.Filter.Lt(x => x.CreatedTime, threshold);
+
+        var contentDocsToDelete = await _dc.ContentLogs.Find(contentLogFilter).Limit(batchSize).Project(x => x.Id).ToListAsync();
+        long contentDeletedCount = 0;
+        if (contentDocsToDelete.Any())
+        {
+            var deleteFilter = Builders<ConversationContentLogDocument>.Filter.In(x => x.Id, contentDocsToDelete);
+            var contentDeleted = await _dc.ContentLogs.DeleteManyAsync(deleteFilter);
+            contentDeletedCount = contentDeleted.DeletedCount;
+        }
+
+        var stateDocsToDelete = await _dc.StateLogs.Find(stateLogFilter).Limit(batchSize).Project(x => x.Id).ToListAsync();
+        long stateDeletedCount = 0;
+        if (stateDocsToDelete.Any())
+        {
+            var deleteFilter = Builders<ConversationStateLogDocument>.Filter.In(x => x.Id, stateDocsToDelete);
+            var stateDeleted = await _dc.StateLogs.DeleteManyAsync(deleteFilter);
+            stateDeletedCount = stateDeleted.DeletedCount;
+        }
+
+        return (int)(contentDeletedCount + stateDeletedCount);
+    }
+    #endregion
+
     #region Instruction Log
     public async Task<bool> SaveInstructionLogs(IEnumerable<InstructionLogModel> logs)
     {

src/WebStarter/appsettings.json

@@ -707,6 +707,8 @@
       "BatchSize": 50,
       "MessageLimit": 2,
       "BufferHours": 12,
+      "LogRetentionDays": 30,
+      "LogBatchSize": 2000,
       "ExcludeAgentIds": []
     },
     "RateLimit": {