From ea5d44a0edbda6ffa0351296bb43406ebd68e18e Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Wed, 25 Mar 2026 11:17:31 +0800 Subject: [PATCH 1/3] Enhance crontab job execution with re-entry protection --- .../Crontab/ICrontabService.cs | 1 + .../Crontab/Models/CrontabItem.cs | 3 + .../Services/CrontabService.cs | 59 +++++++++++++++++++ .../Controllers/Crontab/CrontabController.cs | 21 ++----- 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabService.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabService.cs index 97ef728f5..025685f96 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabService.cs @@ -4,4 +4,5 @@ public interface ICrontabService { Task> GetCrontable(); Task ScheduledTimeArrived(CrontabItem item); + Task ExecuteTimeArrivedItemWithReentryProtection(CrontabItem item); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs index 227ca99ef..538e04db6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs @@ -32,6 +32,9 @@ public class CrontabItem : ScheduleTaskArgs [JsonPropertyName("trigger_type")] public CronTabItemTriggerType TriggerType { get; set; } = CronTabItemTriggerType.BackgroundWatcher; + [JsonPropertyName("reentry_protection")] + public bool ReentryProtection { get; set; } = true; + public override string ToString() { return $"{Title}: {Description} [AgentId: {AgentId}, UserId: {UserId}]"; diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index c8838b334..c4d1cad51 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -15,6 +15,7 @@ limitations under the License. ******************************************************************************/ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Tasks; @@ -127,4 +128,62 @@ await HookEmitter.Emit(_services, async hook => } }, item.AgentId); } + + public async Task ExecuteTimeArrivedItemWithReentryProtection(CrontabItem item) + { + if (!item.ReentryProtection) + { + await ExecuteTimeArrivedItem(item); + return; + } + + var lockKey = $"crontab:execution:{item.Title}"; + using var scope = _services.CreateScope(); + var locker = scope.ServiceProvider.GetRequiredService(); + var acquired = false; + var lockAcquired = false; + + try + { + acquired = await locker.LockAsync(lockKey, async () => + { + lockAcquired = true; + _logger.LogInformation("Crontab: {0}, Distributed lock acquired, beginning execution...", item.Title); + await ExecuteTimeArrivedItem(item); + }, timeout: 600); + + if (!acquired) + { + _logger.LogWarning("Crontab: {0}, Failed to acquire distributed lock, task is still executing, skipping this occurrence to prevent re-entry.", item.Title); + } + } + catch (Exception ex) + { + if (!lockAcquired) + { + _logger.LogWarning("Crontab: {0}, Redis exception occurred before acquiring lock: {1}, executing without lock protection (re-entry protection disabled).", item.Title, ex.Message); + await ExecuteTimeArrivedItem(item); + } + else + { + _logger.LogWarning("Crontab: {0}, Redis exception occurred after lock acquired: {1}, task execution completed but lock release failed.", item.Title, ex.Message); + } + } + } + + private async Task ExecuteTimeArrivedItem(CrontabItem item) + { + try + { + _logger.LogInformation($"Start running crontab {item.Title}"); + await ScheduledTimeArrived(item); + _logger.LogInformation($"Complete running crontab {item.Title}"); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when running crontab {item.Title}"); + return false; + } + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs index 09e4cf9e3..bc5f98b93 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs @@ -60,7 +60,7 @@ public async Task SchedulingCrontab() { if (item.CheckNextOccurrenceEveryOneMinute()) { - _logger.LogInformation("Crontab: {0}, One occurrence was matched, Beginning execution...", item.Title); + _logger.LogInformation($"Crontab: {item.Title}, One occurrence was matched, attempting to execute..."); Task.Run(() => ExecuteTimeArrivedItem(item, _services)); result.OccurrenceMatchedItems.Add(item.Title); } @@ -84,21 +84,10 @@ private async Task> GetCrontabItems(string? title = null) return allowedCrons.Where(cron => cron.Title.IsEqualTo(title)).ToList(); } - private async Task ExecuteTimeArrivedItem(CrontabItem item, IServiceProvider services) + private async Task ExecuteTimeArrivedItem(CrontabItem item, IServiceProvider services) { - try - { - using var scope = services.CreateScope(); - var crontabService = scope.ServiceProvider.GetRequiredService(); - _logger.LogInformation($"Start running crontab {item.Title}"); - await crontabService.ScheduledTimeArrived(item); - _logger.LogInformation($"Complete running crontab {item.Title}"); - return true; - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error when running crontab {item.Title}"); - return false; - } + using var scope = services.CreateScope(); + var crontabService = scope.ServiceProvider.GetRequiredService(); + await crontabService.ExecuteTimeArrivedItemWithReentryProtection(item); } } From 0c67ec9ec37c6a1b6df4691713159362a67a934b Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Wed, 1 Apr 2026 09:45:20 +0800 Subject: [PATCH 2/3] Enhance CrontabService to log warnings for disabled triggers and update CrontabItemDocument to include TriggerType and ReentryProtection properties. --- .../BotSharp.Core.Crontab/Services/CrontabService.cs | 7 ++++++- .../Collections/CrontabItemDocument.cs | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 850ab4eab..8913800fb 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -117,7 +117,12 @@ public async Task ScheduledTimeArrived(CrontabItem item) { _logger.LogDebug($"ScheduledTimeArrived {item}"); - if (!await HasEnabledTriggerRule(item)) return; + var triggerEnabled = await HasEnabledTriggerRule(item); + if (!triggerEnabled) + { + _logger.LogWarning("Crontab: {0}, Trigger is not enabled, skipping this occurrence.", item.Title); + return; + } await HookEmitter.Emit(_services, async hook => { diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs index 17a49c2bc..ee5858ce4 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs @@ -18,6 +18,8 @@ public class CrontabItemDocument : MongoBase public bool LessThan60Seconds { get; set; } = false; public IEnumerable Tasks { get; set; } = []; public DateTime CreatedTime { get; set; } = DateTime.UtcNow; + public int TriggerType { get; set; } + public bool ReentryProtection { get; set; } = true; public static CrontabItem ToDomainModel(CrontabItemDocument item) { @@ -36,7 +38,9 @@ public static CrontabItem ToDomainModel(CrontabItemDocument item) LastExecutionTime = item.LastExecutionTime, LessThan60Seconds = item.LessThan60Seconds, Tasks = item.Tasks?.Select(x => CronTaskMongoElement.ToDomainElement(x))?.ToArray() ?? [], - CreatedTime = item.CreatedTime + CreatedTime = item.CreatedTime, + TriggerType = (CronTabItemTriggerType)item.TriggerType, + ReentryProtection = item.ReentryProtection }; } @@ -57,7 +61,9 @@ public static CrontabItemDocument ToMongoModel(CrontabItem item) LastExecutionTime = item.LastExecutionTime, LessThan60Seconds = item.LessThan60Seconds, Tasks = item.Tasks?.Select(x => CronTaskMongoElement.ToMongoElement(x))?.ToList() ?? [], - CreatedTime = item.CreatedTime + CreatedTime = item.CreatedTime, + TriggerType = (int)item.TriggerType, + ReentryProtection = item.ReentryProtection }; } } From d7f14d2885af4a121646e214877f73154bac74d3 Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Thu, 2 Apr 2026 14:58:13 +0800 Subject: [PATCH 3/3] Using IServiceScopeFactory instead of IServiceProvider to create ServiceScope; Set default value to false for perporty ReentryProtection --- .../BotSharp.Abstraction/Crontab/Models/CrontabItem.cs | 2 +- .../BotSharp.Core.Crontab/Services/CrontabService.cs | 6 ++++-- .../Controllers/Crontab/CrontabController.cs | 9 ++++++--- .../Collections/CrontabItemDocument.cs | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs index 538e04db6..71bc20985 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs @@ -33,7 +33,7 @@ public class CrontabItem : ScheduleTaskArgs public CronTabItemTriggerType TriggerType { get; set; } = CronTabItemTriggerType.BackgroundWatcher; [JsonPropertyName("reentry_protection")] - public bool ReentryProtection { get; set; } = true; + public bool ReentryProtection { get; set; } public override string ToString() { diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 8913800fb..22d5f9c9d 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -34,11 +34,13 @@ namespace BotSharp.Core.Crontab.Services; public class CrontabService : ICrontabService, ITaskFeeder { private readonly IServiceProvider _services; + private readonly IServiceScopeFactory _scopeFactory; private ILogger _logger; - public CrontabService(IServiceProvider services, ILogger logger) + public CrontabService(IServiceProvider services, IServiceScopeFactory scopeFactory, ILogger logger) { _services = services; + _scopeFactory = scopeFactory; _logger = logger; } @@ -166,7 +168,7 @@ public async Task ExecuteTimeArrivedItemWithReentryProtection(CrontabItem item) } var lockKey = $"crontab:execution:{item.Title}"; - using var scope = _services.CreateScope(); + using var scope = _scopeFactory.CreateScope(); var locker = scope.ServiceProvider.GetRequiredService(); var acquired = false; var lockAcquired = false; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs index bc5f98b93..41e30df5c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs @@ -10,13 +10,16 @@ namespace BotSharp.OpenAPI.Controllers; public class CrontabController : ControllerBase { private readonly IServiceProvider _services; + private readonly IServiceScopeFactory _scopeFactory; private readonly ILogger _logger; public CrontabController( IServiceProvider services, + IServiceScopeFactory scopeFactory, ILogger logger) { _services = services; + _scopeFactory = scopeFactory; _logger = logger; } @@ -61,7 +64,7 @@ public async Task SchedulingCrontab() if (item.CheckNextOccurrenceEveryOneMinute()) { _logger.LogInformation($"Crontab: {item.Title}, One occurrence was matched, attempting to execute..."); - Task.Run(() => ExecuteTimeArrivedItem(item, _services)); + Task.Run(() => ExecuteTimeArrivedItem(item)); result.OccurrenceMatchedItems.Add(item.Title); } } @@ -84,9 +87,9 @@ private async Task> GetCrontabItems(string? title = null) return allowedCrons.Where(cron => cron.Title.IsEqualTo(title)).ToList(); } - private async Task ExecuteTimeArrivedItem(CrontabItem item, IServiceProvider services) + private async Task ExecuteTimeArrivedItem(CrontabItem item) { - using var scope = services.CreateScope(); + using var scope = _scopeFactory.CreateScope(); var crontabService = scope.ServiceProvider.GetRequiredService(); await crontabService.ExecuteTimeArrivedItemWithReentryProtection(item); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs index ee5858ce4..9bf332d61 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs @@ -19,7 +19,7 @@ public class CrontabItemDocument : MongoBase public IEnumerable Tasks { get; set; } = []; public DateTime CreatedTime { get; set; } = DateTime.UtcNow; public int TriggerType { get; set; } - public bool ReentryProtection { get; set; } = true; + public bool ReentryProtection { get; set; } public static CrontabItem ToDomainModel(CrontabItemDocument item) {