Skip to content

Commit 7c4d0db

Browse files
yileicnyileicn
authored
Merge pull request #1317 from adenchen123/GTP-12762
Enhance crontab job execution with re-entry protection
2 parents 39a7421 + d7f14d2 commit 7c4d0db

File tree

5 files changed

+89
-21
lines changed

5 files changed

+89
-21
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabService.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ public interface ICrontabService
44
{
55
Task<List<CrontabItem>> GetCrontable();
66
Task ScheduledTimeArrived(CrontabItem item);
7+
Task ExecuteTimeArrivedItemWithReentryProtection(CrontabItem item);
78
}

‎src/Infrastructure/BotSharp.Abstraction/Crontab/Models/CrontabItem.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ public class CrontabItem : ScheduleTaskArgs
3232
[JsonPropertyName("trigger_type")]
3333
public CronTabItemTriggerType TriggerType { get; set; } = CronTabItemTriggerType.BackgroundWatcher;
3434

35+
[JsonPropertyName("reentry_protection")]
36+
public bool ReentryProtection { get; set; }
37+
3538
public override string ToString()
3639
{
3740
return $"{Title}: {Description} [AgentId: {AgentId}, UserId: {UserId}]";

‎src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515
******************************************************************************/
1616

1717
using BotSharp.Abstraction.Agents.Models;
18+
using BotSharp.Abstraction.Infrastructures;
1819
using BotSharp.Abstraction.Repositories;
1920
using BotSharp.Abstraction.Repositories.Filters;
2021
using BotSharp.Abstraction.Tasks;
@@ -33,11 +34,13 @@ namespace BotSharp.Core.Crontab.Services;
3334
public class CrontabService : ICrontabService, ITaskFeeder
3435
{
3536
private readonly IServiceProvider _services;
37+
private readonly IServiceScopeFactory _scopeFactory;
3638
private ILogger _logger;
3739

38-
public CrontabService(IServiceProvider services, ILogger<CrontabService> logger)
40+
public CrontabService(IServiceProvider services, IServiceScopeFactory scopeFactory, ILogger<CrontabService> logger)
3941
{
4042
_services = services;
43+
_scopeFactory = scopeFactory;
4144
_logger = logger;
4245
}
4346

@@ -116,7 +119,12 @@ public async Task ScheduledTimeArrived(CrontabItem item)
116119
{
117120
_logger.LogDebug($"ScheduledTimeArrived {item}");
118121

119-
if (!await HasEnabledTriggerRule(item)) return;
122+
var triggerEnabled = await HasEnabledTriggerRule(item);
123+
if (!triggerEnabled)
124+
{
125+
_logger.LogWarning("Crontab: {0}, Trigger is not enabled, skipping this occurrence.", item.Title);
126+
return;
127+
}
120128

121129
await HookEmitter.Emit<ICrontabHook>(_services, async hook =>
122130
{
@@ -150,4 +158,62 @@ private async Task<bool> HasEnabledTriggerRule(CrontabItem item)
150158
// Opt-out only: block when a matching trigger rule exists and Disabled is true.
151159
return !agent.Rules.Any(r => r.TriggerName == item.Title && r.Disabled);
152160
}
161+
162+
public async Task ExecuteTimeArrivedItemWithReentryProtection(CrontabItem item)
163+
{
164+
if (!item.ReentryProtection)
165+
{
166+
await ExecuteTimeArrivedItem(item);
167+
return;
168+
}
169+
170+
var lockKey = $"crontab:execution:{item.Title}";
171+
using var scope = _scopeFactory.CreateScope();
172+
var locker = scope.ServiceProvider.GetRequiredService<IDistributedLocker>();
173+
var acquired = false;
174+
var lockAcquired = false;
175+
176+
try
177+
{
178+
acquired = await locker.LockAsync(lockKey, async () =>
179+
{
180+
lockAcquired = true;
181+
_logger.LogInformation("Crontab: {0}, Distributed lock acquired, beginning execution...", item.Title);
182+
await ExecuteTimeArrivedItem(item);
183+
}, timeout: 600);
184+
185+
if (!acquired)
186+
{
187+
_logger.LogWarning("Crontab: {0}, Failed to acquire distributed lock, task is still executing, skipping this occurrence to prevent re-entry.", item.Title);
188+
}
189+
}
190+
catch (Exception ex)
191+
{
192+
if (!lockAcquired)
193+
{
194+
_logger.LogWarning("Crontab: {0}, Redis exception occurred before acquiring lock: {1}, executing without lock protection (re-entry protection disabled).", item.Title, ex.Message);
195+
await ExecuteTimeArrivedItem(item);
196+
}
197+
else
198+
{
199+
_logger.LogWarning("Crontab: {0}, Redis exception occurred after lock acquired: {1}, task execution completed but lock release failed.", item.Title, ex.Message);
200+
}
201+
}
202+
}
203+
204+
private async Task<bool> ExecuteTimeArrivedItem(CrontabItem item)
205+
{
206+
try
207+
{
208+
_logger.LogInformation($"Start running crontab {item.Title}");
209+
await ScheduledTimeArrived(item);
210+
_logger.LogInformation($"Complete running crontab {item.Title}");
211+
return true;
212+
}
213+
catch (Exception ex)
214+
{
215+
_logger.LogError(ex, $"Error when running crontab {item.Title}");
216+
return false;
217+
}
218+
}
153219
}

‎src/Infrastructure/BotSharp.OpenAPI/Controllers/Crontab/CrontabController.cs‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@ namespace BotSharp.OpenAPI.Controllers;
1010
public class CrontabController : ControllerBase
1111
{
1212
private readonly IServiceProvider _services;
13+
private readonly IServiceScopeFactory _scopeFactory;
1314
private readonly ILogger<CrontabController> _logger;
1415

1516
public CrontabController(
1617
IServiceProvider services,
18+
IServiceScopeFactory scopeFactory,
1719
ILogger<CrontabController> logger)
1820
{
1921
_services = services;
22+
_scopeFactory = scopeFactory;
2023
_logger = logger;
2124
}
2225

@@ -60,8 +63,8 @@ public async Task<CrontabSchedulingResult> SchedulingCrontab()
6063
{
6164
if (item.CheckNextOccurrenceEveryOneMinute())
6265
{
63-
_logger.LogInformation("Crontab: {0}, One occurrence was matched, Beginning execution...", item.Title);
64-
Task.Run(() => ExecuteTimeArrivedItem(item, _services));
66+
_logger.LogInformation($"Crontab: {item.Title}, One occurrence was matched, attempting to execute...");
67+
Task.Run(() => ExecuteTimeArrivedItem(item));
6568
result.OccurrenceMatchedItems.Add(item.Title);
6669
}
6770
}
@@ -84,21 +87,10 @@ private async Task<List<CrontabItem>> GetCrontabItems(string? title = null)
8487
return allowedCrons.Where(cron => cron.Title.IsEqualTo(title)).ToList();
8588
}
8689

87-
private async Task<bool> ExecuteTimeArrivedItem(CrontabItem item, IServiceProvider services)
90+
private async Task ExecuteTimeArrivedItem(CrontabItem item)
8891
{
89-
try
90-
{
91-
using var scope = services.CreateScope();
92-
var crontabService = scope.ServiceProvider.GetRequiredService<ICrontabService>();
93-
_logger.LogInformation($"Start running crontab {item.Title}");
94-
await crontabService.ScheduledTimeArrived(item);
95-
_logger.LogInformation($"Complete running crontab {item.Title}");
96-
return true;
97-
}
98-
catch (Exception ex)
99-
{
100-
_logger.LogError(ex, $"Error when running crontab {item.Title}");
101-
return false;
102-
}
92+
using var scope = _scopeFactory.CreateScope();
93+
var crontabService = scope.ServiceProvider.GetRequiredService<ICrontabService>();
94+
await crontabService.ExecuteTimeArrivedItemWithReentryProtection(item);
10395
}
10496
}

‎src/Plugins/BotSharp.Plugin.MongoStorage/Collections/CrontabItemDocument.cs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public class CrontabItemDocument : MongoBase
1818
public bool LessThan60Seconds { get; set; } = false;
1919
public IEnumerable<CronTaskMongoElement> Tasks { get; set; } = [];
2020
public DateTime CreatedTime { get; set; } = DateTime.UtcNow;
21+
public int TriggerType { get; set; }
22+
public bool ReentryProtection { get; set; }
2123

2224
public static CrontabItem ToDomainModel(CrontabItemDocument item)
2325
{
@@ -36,7 +38,9 @@ public static CrontabItem ToDomainModel(CrontabItemDocument item)
3638
LastExecutionTime = item.LastExecutionTime,
3739
LessThan60Seconds = item.LessThan60Seconds,
3840
Tasks = item.Tasks?.Select(x => CronTaskMongoElement.ToDomainElement(x))?.ToArray() ?? [],
39-
CreatedTime = item.CreatedTime
41+
CreatedTime = item.CreatedTime,
42+
TriggerType = (CronTabItemTriggerType)item.TriggerType,
43+
ReentryProtection = item.ReentryProtection
4044
};
4145
}
4246

@@ -57,7 +61,9 @@ public static CrontabItemDocument ToMongoModel(CrontabItem item)
5761
LastExecutionTime = item.LastExecutionTime,
5862
LessThan60Seconds = item.LessThan60Seconds,
5963
Tasks = item.Tasks?.Select(x => CronTaskMongoElement.ToMongoElement(x))?.ToList() ?? [],
60-
CreatedTime = item.CreatedTime
64+
CreatedTime = item.CreatedTime,
65+
TriggerType = (int)item.TriggerType,
66+
ReentryProtection = item.ReentryProtection
6167
};
6268
}
6369
}

0 commit comments

Comments
 (0)