Skip to content

Commit 283ee66

Browse files
author
Timothy Dodd
committed
Enhance filtering, memory, and chart functionality
- Added debounce logic to `BatchingService` for efficient batch processing. - Introduced tri-state filtering for log levels and pods. - Implemented `MemoryManagementService` for log retention and auto-cleanup. - Added zoom controls to charts using `chartjs-plugin-zoom`. - Replaced `@iharbeck/ngx-virtual-scroller` with a custom virtual scroller. - Enhanced `LogApiService` to support exclude filters. - Updated dropdowns and context menus for better usability. - Improved mobile responsiveness and styling. - Updated dependencies and removed `@iharbeck/ngx-virtual-scroller`. - Refactored `LogViewportComponent` for memory and filtering integration. - Updated documentation to reflect new features.
1 parent 848716e commit 283ee66

20 files changed

Lines changed: 2931 additions & 331 deletions

‎src/LogMkAgent/Services/BatchService.cs‎

Lines changed: 85 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@ public class BatchingService : IDisposable
1212
private readonly ILogger<BatchingService> _logger;
1313
private readonly BatchingOptions _options;
1414
private readonly SettingsService _settingsService;
15-
private readonly Timer _timer;
15+
private Timer? _debounceTimer;
1616
private readonly SemaphoreSlim _sendSemaphore = new(1, 1);
1717
private readonly CancellationTokenSource _cancellationTokenSource = new();
18+
private readonly object _timerLock = new();
1819

1920
// Retry mechanism
2021
private readonly Queue<BatchItem> _retryQueue = new();
@@ -34,6 +35,11 @@ public class BatchingService : IDisposable
3435
private long _totalValidationFailures;
3536
private DateTime _lastSuccessfulSend = DateTime.UtcNow;
3637

38+
// Debounce configuration
39+
private readonly TimeSpan _debounceDelay = TimeSpan.FromMilliseconds(150); // Send after 150ms of no new logs
40+
private readonly TimeSpan _maxWaitTime = TimeSpan.FromSeconds(2); // Force send after 2 seconds max
41+
private DateTime _firstLogAddedTime = DateTime.MinValue;
42+
3743
private volatile bool _disposed;
3844

3945
public BatchingService(
@@ -49,14 +55,15 @@ public BatchingService(
4955

5056
ValidateOptions();
5157

52-
_timer = new Timer(
53-
async _ => await ProcessBatchAsync().ConfigureAwait(false),
54-
null,
55-
_options.BatchInterval,
56-
_options.BatchInterval);
58+
// Configure debounce delay from options if available, otherwise use defaults
59+
if (_options.BatchInterval < TimeSpan.FromSeconds(5))
60+
{
61+
_debounceDelay = TimeSpan.FromMilliseconds(Math.Min(_options.BatchInterval.TotalMilliseconds / 10, 150));
62+
_maxWaitTime = _options.BatchInterval;
63+
}
5764

58-
_logger.LogInformation("BatchingService initialized with interval: {Interval}, max size: {MaxSize}, timeout: {Timeout}",
59-
_options.BatchInterval, _options.MaxBatchSize, _options.SendTimeout);
65+
_logger.LogInformation("BatchingService initialized with debounce: {DebounceDelay}ms, max wait: {MaxWait}s, max size: {MaxSize}, timeout: {Timeout}",
66+
_debounceDelay.TotalMilliseconds, _maxWaitTime.TotalSeconds, _options.MaxBatchSize, _options.SendTimeout);
6067
}
6168

6269
private void ValidateOptions()
@@ -87,12 +94,69 @@ public void AddData(LogLine data)
8794

8895
_batchData.Enqueue(data);
8996

97+
// Track when first log was added if this is the first in a batch
98+
if (_firstLogAddedTime == DateTime.MinValue)
99+
{
100+
_firstLogAddedTime = DateTime.UtcNow;
101+
}
102+
90103
// Trigger immediate send if batch is full
91104
if (_batchData.Count >= _options.MaxBatchSize)
92105
{
93-
_ = Task.Run(async () => await ProcessBatchAsync().ConfigureAwait(false),
94-
_cancellationTokenSource.Token);
106+
TriggerImmediateSend();
107+
}
108+
else
109+
{
110+
// Reset the debounce timer
111+
ResetDebounceTimer();
112+
}
113+
}
114+
115+
private void ResetDebounceTimer()
116+
{
117+
lock (_timerLock)
118+
{
119+
// Calculate how long we've been waiting
120+
var waitTime = DateTime.UtcNow - _firstLogAddedTime;
121+
122+
// If we've been waiting too long, send immediately
123+
if (_firstLogAddedTime != DateTime.MinValue && waitTime >= _maxWaitTime)
124+
{
125+
TriggerImmediateSend();
126+
return;
127+
}
128+
129+
// Create timer if it doesn't exist, otherwise reset it
130+
if (_debounceTimer == null)
131+
{
132+
_debounceTimer = new Timer(
133+
_ => TriggerImmediateSend(),
134+
null,
135+
_debounceDelay,
136+
Timeout.InfiniteTimeSpan);
137+
}
138+
else
139+
{
140+
// Reuse existing timer by changing its due time
141+
_debounceTimer.Change(_debounceDelay, Timeout.InfiniteTimeSpan);
142+
}
143+
}
144+
}
145+
146+
private void TriggerImmediateSend()
147+
{
148+
lock (_timerLock)
149+
{
150+
_debounceTimer?.Dispose();
151+
_debounceTimer = null;
95152
}
153+
154+
_ = Task.Run(async () =>
155+
{
156+
await ProcessBatchAsync().ConfigureAwait(false);
157+
// Reset first log time after processing
158+
_firstLogAddedTime = DateTime.MinValue;
159+
}, _cancellationTokenSource.Token);
96160
}
97161

98162
private async Task ProcessBatchAsync()
@@ -323,8 +387,12 @@ public async Task FlushAsync(CancellationToken cancellationToken = default)
323387
{
324388
_logger.LogInformation("Flushing remaining batches...");
325389

326-
// Stop the timer to prevent new batches from being processed
327-
await _timer.DisposeAsync().ConfigureAwait(false);
390+
// Stop the debounce timer to prevent new batches from being processed
391+
lock (_timerLock)
392+
{
393+
_debounceTimer?.Dispose();
394+
_debounceTimer = null;
395+
}
328396

329397
// Process all remaining data
330398
while (!_batchData.IsEmpty || HasPendingRetries())
@@ -394,7 +462,10 @@ public void Dispose()
394462
}
395463
finally
396464
{
397-
_timer?.Dispose();
465+
lock (_timerLock)
466+
{
467+
_debounceTimer?.Dispose();
468+
}
398469
_sendSemaphore?.Dispose();
399470
_cancellationTokenSource?.Dispose();
400471
}
@@ -410,7 +481,7 @@ private class BatchItem
410481

411482
public class BatchingOptions
412483
{
413-
public TimeSpan BatchInterval { get; set; } = TimeSpan.FromSeconds(10);
484+
public TimeSpan BatchInterval { get; set; } = TimeSpan.FromSeconds(2); // Now acts as max wait time
414485
public int MaxBatchSize { get; set; } = 100;
415486
public TimeSpan SendTimeout { get; set; } = TimeSpan.FromSeconds(30);
416487
}

‎src/LogMkApi/Controllers/LogController.cs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -358,13 +358,16 @@ public async Task<IActionResult> GetLogs(
358358
[FromQuery] DateTime? dateStart = null,
359359
[FromQuery] DateTime? dateEnd = null,
360360
[FromQuery] string? search = null, [FromQuery] string[]? podName = null,
361-
[FromQuery] string[]? deployment = null, [FromQuery] string[]? logLevel = null
361+
[FromQuery] string[]? deployment = null, [FromQuery] string[]? logLevel = null,
362+
[FromQuery] string? excludeSearch = null, [FromQuery] string[]? excludePodName = null,
363+
[FromQuery] string[]? excludeDeployment = null, [FromQuery] string[]? excludeLogLevel = null
362364

363365
)
364366
{
365367
var stats = await _logSummaryRepo.GetStatistics(dateStart,
366368
dateEnd,
367-
search, podName, deployment, logLevel);
369+
search, podName, deployment, logLevel,
370+
excludeSearch, excludePodName, excludeDeployment, excludeLogLevel);
368371
if (stats == null)
369372
return NotFound();
370373

@@ -376,15 +379,18 @@ public async Task<PagedResults<Log>> GetLogs([FromQuery] int page = 1,
376379
[FromQuery] DateTime? dateStart = null,
377380
[FromQuery] DateTime? dateEnd = null,
378381
[FromQuery] string? search = null, [FromQuery] string[]? podName = null,
379-
[FromQuery] string[]? deployment = null, [FromQuery] string[]? logLevel = null
382+
[FromQuery] string[]? deployment = null, [FromQuery] string[]? logLevel = null,
383+
[FromQuery] string? excludeSearch = null, [FromQuery] string[]? excludePodName = null,
384+
[FromQuery] string[]? excludeDeployment = null, [FromQuery] string[]? excludeLogLevel = null
380385

381386
)
382387
{
383388
var entries = await _logRepo.GetAll((page - 1) * pageSize,
384389
pageSize,
385390
dateStart,
386391
dateEnd,
387-
search, podName, deployment, logLevel);
392+
search, podName, deployment, logLevel,
393+
excludeSearch, excludePodName, excludeDeployment, excludeLogLevel);
388394
return entries;
389395
}
390396
[AllowAnonymous]

‎src/LogMkApi/Data/LogRepo.cs‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,28 @@ private void AddMany<T>(IEnumerable<T>? items, DynamicParameters dynamicParamete
5353
builder.AppendAnd($"{fieldClause} IN ({ids})");
5454
}
5555
}
56+
57+
private void AddManyExclude<T>(IEnumerable<T>? items, DynamicParameters dynamicParameters, WhereBuilder builder, string paramName, string fieldClause)
58+
{
59+
if (items != null && items.Any())
60+
{
61+
List<string> keys = dynamicParameters.AddList(items, paramName);
62+
var ids = string.Join(',', keys);
63+
builder.AppendAnd($"{fieldClause} NOT IN ({ids})");
64+
}
65+
}
5666
public async Task<PagedResults<Log>> GetAll(int offset = 0,
5767
int pageSize = 100,
5868
DateTime? dateStart = null,
5969
DateTime? dateEnd = null,
6070
string? search = null,
6171
string[]? pod = null,
6272
string[]? deployment = null,
63-
string[]? logLevel = null)
73+
string[]? logLevel = null,
74+
string? excludeSearch = null,
75+
string[]? excludePod = null,
76+
string[]? excludeDeployment = null,
77+
string[]? excludeLogLevel = null)
6478
{
6579
var result = new PagedResults<Log>();
6680
var sortOrderBuilder = new List<string>();
@@ -69,14 +83,22 @@ public async Task<PagedResults<Log>> GetAll(int offset = 0,
6983

7084
var dynamicParameters = new DynamicParameters();
7185

86+
// Include filters
7287
AddMany(pod, dynamicParameters, whereBuilder, "plp", "l.Pod");
7388
AddMany(deployment, dynamicParameters, whereBuilder, "dlp", "l.Deployment");
7489
AddMany(logLevel, dynamicParameters, whereBuilder, "llp", "l.LogLevel");
7590

91+
// Exclude filters
92+
AddManyExclude(excludePod, dynamicParameters, whereBuilder, "eplp", "l.Pod");
93+
AddManyExclude(excludeDeployment, dynamicParameters, whereBuilder, "edlp", "l.Deployment");
94+
AddManyExclude(excludeLogLevel, dynamicParameters, whereBuilder, "ellp", "l.LogLevel");
7695

7796
if (!string.IsNullOrWhiteSpace(search))
7897
search = $"%{search}%";
7998

99+
if (!string.IsNullOrWhiteSpace(excludeSearch))
100+
excludeSearch = $"%{excludeSearch}%";
101+
80102

81103

82104
dynamicParameters.AddIfNotNull("offset", offset);
@@ -87,17 +109,25 @@ public async Task<PagedResults<Log>> GetAll(int offset = 0,
87109
dynamicParameters.AddIfNotNull("dateEndHour", dateEnd?.Hour);
88110

89111
dynamicParameters.AddIfNotNull("search", search);
112+
dynamicParameters.AddIfNotNull("excludeSearch", excludeSearch);
90113

91114
whereBuilder.AppendAnd(dateStart, "l.LogDate >= @dateStart AND (l.LogDate != @dateStart || (l.LogHour >= @dateStartHour))");
92115
whereBuilder.AppendAnd(dateEnd, "l.LogDate <= @dateEnd AND (l.LogDate != @dateEnd || (l.LogHour < @dateEndHour)) ");
93116

117+
// Include search filter
94118
var likeClause = new AndOrBuilder();
95119
likeClause.AppendOr(search, "l.Line LIKE @search ");
96120
if (likeClause.Length > 0)
97121
{
98122
whereBuilder.AppendAnd($"({likeClause})");
99123
}
100124

125+
// Exclude search filter
126+
if (!string.IsNullOrWhiteSpace(excludeSearch))
127+
{
128+
whereBuilder.AppendAnd("l.Line NOT LIKE @excludeSearch");
129+
}
130+
101131
var queryBase = @"
102132
select * from Log l
103133
";

‎src/LogMkApi/Data/LogSummaryRepo.cs‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,26 @@ private void AddMany<T>(IEnumerable<T>? items, DynamicParameters dynamicParamete
2828
}
2929
}
3030

31+
private void AddManyExclude<T>(IEnumerable<T>? items, DynamicParameters dynamicParameters, WhereBuilder builder, string paramName, string fieldClause)
32+
{
33+
if (items != null && items.Any())
34+
{
35+
List<string> keys = dynamicParameters.AddList(items, paramName);
36+
var ids = string.Join(',', keys);
37+
builder.AppendAnd($"{fieldClause} NOT IN ({ids})");
38+
}
39+
}
40+
3141
public async Task<LogStatistic> GetStatistics(DateTime? dateStart = null,
3242
DateTime? dateEnd = null,
3343
string? search = null,
3444
string[]? pod = null,
3545
string[]? deployment = null,
36-
string[]? logLevel = null)
46+
string[]? logLevel = null,
47+
string? excludeSearch = null,
48+
string[]? excludePod = null,
49+
string[]? excludeDeployment = null,
50+
string[]? excludeLogLevel = null)
3751
{
3852

3953
var dynamicParameters = new DynamicParameters();
@@ -48,10 +62,16 @@ public async Task<LogStatistic> GetStatistics(DateTime? dateStart = null,
4862

4963

5064
var whereBuilder = new WhereBuilder();
65+
// Include filters
5166
AddMany(pod, dynamicParameters, whereBuilder, "plp", "Pod");
5267
AddMany(deployment, dynamicParameters, whereBuilder, "dlp", "Deployment");
5368
AddMany(logLevel, dynamicParameters, whereBuilder, "llp", "LogLevel");
5469

70+
// Exclude filters (Note: excludeSearch not supported for stats since summary tables don't have Line content)
71+
AddManyExclude(excludePod, dynamicParameters, whereBuilder, "eplp", "Pod");
72+
AddManyExclude(excludeDeployment, dynamicParameters, whereBuilder, "edlp", "Deployment");
73+
AddManyExclude(excludeLogLevel, dynamicParameters, whereBuilder, "ellp", "LogLevel");
74+
5575
var isGreaterThan3Days = dateStart == null ? true : DateTime.UtcNow.Subtract(dateStart.Value).TotalDays > 3;
5676
var query = "";
5777
if (isGreaterThan3Days)

0 commit comments

Comments
 (0)