diff --git a/ThingConnect.Pulse.Server/Infrastructure/SqliteWalInterceptor.cs b/ThingConnect.Pulse.Server/Infrastructure/SqliteWalInterceptor.cs new file mode 100644 index 0000000..2737545 --- /dev/null +++ b/ThingConnect.Pulse.Server/Infrastructure/SqliteWalInterceptor.cs @@ -0,0 +1,51 @@ +using Microsoft.EntityFrameworkCore.Diagnostics; +using System.Data.Common; + +namespace ThingConnect.Pulse.Server.Infrastructure; + +/// +/// Enables WAL mode and busy_timeout on every SQLite connection to allow concurrent reads +/// during high-frequency writes from the monitoring background service. +/// +/// Without WAL, SQLite's default DELETE journal mode holds an exclusive lock for the +/// duration of each write, causing read timeouts when probes are writing continuously. +/// WAL mode decouples readers from writers entirely. +/// +public sealed class SqliteWalInterceptor : DbConnectionInterceptor +{ + public override void ConnectionOpened(DbConnection connection, ConnectionEndEventData eventData) + { + ApplyPragmas(connection); + } + + public override async Task ConnectionOpenedAsync( + DbConnection connection, + ConnectionEndEventData eventData, + CancellationToken cancellationToken = default) + { + await ApplyPragmasAsync(connection, cancellationToken); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100", Justification = "Pragma SQL is a compile-time constant, not user input.")] + private static void ApplyPragmas(DbConnection connection) + { + using DbCommand cmd = connection.CreateCommand(); + cmd.CommandText = BuildPragmaSql(); + cmd.ExecuteNonQuery(); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100", Justification = "Pragma SQL is a compile-time constant, not user input.")] + private static async Task ApplyPragmasAsync(DbConnection connection, CancellationToken ct) + { + await using DbCommand cmd = connection.CreateCommand(); + cmd.CommandText = BuildPragmaSql(); + await cmd.ExecuteNonQueryAsync(ct); + } + + private static string BuildPragmaSql() => """ + PRAGMA journal_mode=WAL; + PRAGMA busy_timeout=5000; + PRAGMA synchronous=NORMAL; + PRAGMA cache_size=-8000; + """; +} diff --git a/ThingConnect.Pulse.Server/Program.cs b/ThingConnect.Pulse.Server/Program.cs index 75aac7d..95873c2 100644 --- a/ThingConnect.Pulse.Server/Program.cs +++ b/ThingConnect.Pulse.Server/Program.cs @@ -48,7 +48,8 @@ public static async Task Main(string[] args) // Add services to the container. builder.Services.AddDbContext(options => - options.UseSqlite(builder.Configuration.GetConnectionString("DefaultConnection"))); + options.UseSqlite(builder.Configuration.GetConnectionString("DefaultConnection")) + .AddInterceptors(new SqliteWalInterceptor())); // Configure Identity and Authentication builder.Services.AddIdentity(options => @@ -86,7 +87,7 @@ public static async Task Main(string[] args) options.Cookie.HttpOnly = true; options.Cookie.SecurePolicy = builder.Environment.IsDevelopment() ? CookieSecurePolicy.SameAsRequest - : CookieSecurePolicy.Always; + : CookieSecurePolicy.None; options.Cookie.SameSite = SameSiteMode.Lax; options.Cookie.Name = "ThingConnect.Pulse.Auth"; options.Events.OnRedirectToLogin = context => @@ -150,6 +151,9 @@ public static async Task Main(string[] args) builder.Services.AddSingleton(); // Add monitoring services + builder.Services.AddSingleton(); + builder.Services.AddSingleton(p => p.GetRequiredService()); + builder.Services.AddHostedService(p => p.GetRequiredService()); builder.Services.AddScoped(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -164,6 +168,7 @@ public static async Task Main(string[] args) // Add prune services builder.Services.AddScoped(); + builder.Services.AddHostedService(); // Add log cleanup service builder.Services.AddHostedService(); diff --git a/ThingConnect.Pulse.Server/Services/HistoryService.cs b/ThingConnect.Pulse.Server/Services/HistoryService.cs index 6c1c224..86b4eda 100644 --- a/ThingConnect.Pulse.Server/Services/HistoryService.cs +++ b/ThingConnect.Pulse.Server/Services/HistoryService.cs @@ -84,14 +84,9 @@ private async Task> GetRawDataAsync(Guid endpointId, DateTimeO long fromUnix = UnixTimestamp.ToUnixSeconds(from); long toUnix = UnixTimestamp.ToUnixSeconds(to); - // SQLite limitation: fetch all data and filter in memory - var rawData = await _context.CheckResultsRaw - .Where(c => c.EndpointId == endpointId) - .Select(c => new { c.Ts, c.Status, c.RttMs, c.Error }) - .ToListAsync(); - - return rawData - .Where(c => c.Ts >= fromUnix && c.Ts <= toUnix) + return await _context.CheckResultsRaw + .Where(c => c.EndpointId == endpointId && c.Ts >= fromUnix && c.Ts <= toUnix) + .AsNoTracking() .OrderBy(c => c.Ts) .Select(c => new RawCheckDto { @@ -100,7 +95,7 @@ private async Task> GetRawDataAsync(Guid endpointId, DateTimeO RttMs = c.RttMs, Error = c.Error }) - .ToList(); + .ToListAsync(); } private async Task> GetRollup15mDataAsync(Guid endpointId, DateTimeOffset from, DateTimeOffset to) @@ -108,14 +103,9 @@ private async Task> GetRollup15mDataAsync(Guid endpointId, long fromUnix = UnixTimestamp.ToUnixSeconds(from); long toUnix = UnixTimestamp.ToUnixSeconds(to); - // SQLite limitation: fetch all data and filter in memory - var rollupData = await _context.Rollups15m - .Where(r => r.EndpointId == endpointId) - .Select(r => new { r.BucketTs, r.UpPct, r.AvgRttMs, r.DownEvents }) - .ToListAsync(); - - return rollupData - .Where(r => r.BucketTs >= fromUnix && r.BucketTs <= toUnix) + return await _context.Rollups15m + .Where(r => r.EndpointId == endpointId && r.BucketTs >= fromUnix && r.BucketTs <= toUnix) + .AsNoTracking() .OrderBy(r => r.BucketTs) .Select(r => new RollupBucketDto { @@ -124,7 +114,7 @@ private async Task> GetRollup15mDataAsync(Guid endpointId, AvgRttMs = r.AvgRttMs, DownEvents = r.DownEvents }) - .ToList(); + .ToListAsync(); } private async Task> GetRollupDailyDataAsync(Guid endpointId, DateTimeOffset from, DateTimeOffset to) @@ -133,14 +123,9 @@ private async Task> GetRollupDailyDataAsync(Guid endpointId long fromUnix = UnixTimestamp.ToUnixDate(DateOnly.FromDateTime(from.Date)); long toUnix = UnixTimestamp.ToUnixDate(DateOnly.FromDateTime(to.Date)); - // SQLite limitation: fetch all data and filter in memory - var dailyData = await _context.RollupsDaily - .Where(r => r.EndpointId == endpointId) - .Select(r => new { r.BucketDate, r.UpPct, r.AvgRttMs, r.DownEvents }) - .ToListAsync(); - - return dailyData - .Where(r => r.BucketDate >= fromUnix && r.BucketDate <= toUnix) + return await _context.RollupsDaily + .Where(r => r.EndpointId == endpointId && r.BucketDate >= fromUnix && r.BucketDate <= toUnix) + .AsNoTracking() .OrderBy(r => r.BucketDate) .Select(r => new DailyBucketDto { @@ -149,7 +134,7 @@ private async Task> GetRollupDailyDataAsync(Guid endpointId AvgRttMs = r.AvgRttMs, DownEvents = r.DownEvents }) - .ToList(); + .ToListAsync(); } private async Task> GetOutagesAsync(Guid endpointId, DateTimeOffset from, DateTimeOffset to) @@ -157,14 +142,9 @@ private async Task> GetOutagesAsync(Guid endpointId, DateTimeOff long fromUnix = UnixTimestamp.ToUnixSeconds(from); long toUnix = UnixTimestamp.ToUnixSeconds(to); - // SQLite limitation: fetch all data and filter in memory - var outageData = await _context.Outages - .Where(o => o.EndpointId == endpointId) - .Select(o => new { o.StartedTs, o.EndedTs, o.DurationSeconds, o.LastError }) - .ToListAsync(); - - return outageData - .Where(o => o.StartedTs <= toUnix && (o.EndedTs == null || o.EndedTs >= fromUnix)) + return await _context.Outages + .Where(o => o.EndpointId == endpointId && o.StartedTs <= toUnix && (o.EndedTs == null || o.EndedTs >= fromUnix)) + .AsNoTracking() .OrderBy(o => o.StartedTs) .Select(o => new OutageDto { @@ -173,7 +153,7 @@ private async Task> GetOutagesAsync(Guid endpointId, DateTimeOff DurationS = o.DurationSeconds, LastError = o.LastError }) - .ToList(); + .ToListAsync(); } private EndpointDto MapToEndpointDto(Data.Endpoint endpoint) diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/CheckResultWriteQueue.cs b/ThingConnect.Pulse.Server/Services/Monitoring/CheckResultWriteQueue.cs new file mode 100644 index 0000000..62aa801 --- /dev/null +++ b/ThingConnect.Pulse.Server/Services/Monitoring/CheckResultWriteQueue.cs @@ -0,0 +1,122 @@ +using System.Threading.Channels; +using Microsoft.EntityFrameworkCore; +using ThingConnect.Pulse.Server.Data; + +namespace ThingConnect.Pulse.Server.Services.Monitoring; + +public interface ICheckResultWriteQueue +{ + void Enqueue(WriteQueueItem item); +} + +public sealed record WriteQueueItem( + CheckResultRaw RawResult, + Guid? EndpointIdForRttUpdate, + double? RttMs); + +/// +/// Serializes all check_result_raw inserts through a single background writer, eliminating +/// the concurrent SaveChangesAsync flood that causes SQLite read timeouts under high probe rates. +/// +public sealed class CheckResultWriteQueue : BackgroundService, ICheckResultWriteQueue +{ + private readonly Channel _channel = Channel.CreateBounded( + new BoundedChannelOptions(10_000) { FullMode = BoundedChannelFullMode.DropOldest }); + + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public CheckResultWriteQueue(IServiceProvider serviceProvider, ILogger logger) + { + _serviceProvider = serviceProvider; + _logger = logger; + } + + public void Enqueue(WriteQueueItem item) => _channel.Writer.TryWrite(item); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Check result write queue started"); + + var batch = new List(100); + + while (!stoppingToken.IsCancellationRequested) + { + batch.Clear(); + + try + { + // Block until at least one item is available + if (!await _channel.Reader.WaitToReadAsync(stoppingToken)) + { + break; + } + + // Drain all ready items up to batch limit (no extra waiting) + while (batch.Count < 100 && _channel.Reader.TryRead(out WriteQueueItem? item)) + { + batch.Add(item); + } + + await FlushBatchAsync(batch, stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error flushing check result batch of {Count} items", batch.Count); + // Brief pause to avoid tight error loop, then continue + await Task.Delay(1000, stoppingToken).ConfigureAwait(false); + } + } + + // Drain remaining items on shutdown + batch.Clear(); + while (_channel.Reader.TryRead(out WriteQueueItem? item)) + { + batch.Add(item); + } + + if (batch.Count > 0) + { + try + { + await FlushBatchAsync(batch, CancellationToken.None); + _logger.LogInformation("Flushed {Count} remaining check results on shutdown", batch.Count); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to flush remaining {Count} check results on shutdown", batch.Count); + } + } + + _logger.LogInformation("Check result write queue stopped"); + } + + private async Task FlushBatchAsync(List batch, CancellationToken ct) + { + using IServiceScope scope = _serviceProvider.CreateScope(); + PulseDbContext context = scope.ServiceProvider.GetRequiredService(); + + // Bulk insert all check results in one round-trip + context.CheckResultsRaw.AddRange(batch.Select(i => i.RawResult)); + await context.SaveChangesAsync(ct); + + // Update LastRttMs with direct UPDATE statements — no entity load needed + var rttUpdates = batch + .Where(i => i.EndpointIdForRttUpdate.HasValue && i.RttMs.HasValue) + .GroupBy(i => i.EndpointIdForRttUpdate!.Value) + .Select(g => (EndpointId: g.Key, RttMs: g.Last().RttMs!.Value)); + + foreach ((Guid endpointId, double rttMs) in rttUpdates) + { + await context.Endpoints + .Where(e => e.Id == endpointId) + .ExecuteUpdateAsync(s => s.SetProperty(e => e.LastRttMs, rttMs), ct); + } + + _logger.LogDebug("Flushed batch of {Count} check results", batch.Count); + } +} diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs b/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs index 44a97fe..6b3d7e0 100644 --- a/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs +++ b/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs @@ -1,3 +1,4 @@ +using System.Linq; using System.Net; using System.Net.Sockets; using System.Text.RegularExpressions; @@ -47,7 +48,7 @@ public IEnumerable ExpandCidr(string cidr) } byte[] addressBytes = ipAddress.GetAddressBytes(); - uint addressInt = BitConverter.ToUInt32(addressBytes.Reverse().ToArray(), 0); + uint addressInt = BitConverter.ToUInt32(addressBytes, 0); int hostBits = 32 - prefixLength; uint hostCount = (uint)(1 << hostBits); @@ -59,7 +60,8 @@ public IEnumerable ExpandCidr(string cidr) for (uint address = startAddress; address < endAddress && address > networkAddress; address++) { - byte[] bytes = BitConverter.GetBytes(address).Reverse().ToArray(); + byte[] bytes = BitConverter.GetBytes(address); + System.Array.Reverse(bytes); var ip = new IPAddress(bytes); yield return ip.ToString(); } diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs b/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs index bced0e8..053b726 100644 --- a/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs +++ b/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs @@ -15,6 +15,8 @@ public sealed class MonitoringBackgroundService : BackgroundService private readonly SemaphoreSlim _concurrencySemaphore; private readonly ConcurrentDictionary _endpointTimers = new(); private readonly ConcurrentDictionary _probeExecuting = new(); + private readonly ConcurrentDictionary _endpointCache = new(); + private readonly ConcurrentDictionary _endpointIntervals = new(); private readonly int _maxConcurrentProbes; public MonitoringBackgroundService(IServiceProvider serviceProvider, @@ -120,6 +122,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } _endpointTimers.Clear(); + _endpointIntervals.Clear(); _logger.LogInformation("Monitoring background service stopped"); } @@ -134,6 +137,12 @@ private async Task RefreshEndpointsAsync(CancellationToken cancellationToken) .Where(e => e.Enabled) .ToListAsync(cancellationToken); + // Refresh in-memory cache so probes don't need to re-read endpoint data from DB + foreach (Data.Endpoint endpoint in endpoints) + { + _endpointCache[endpoint.Id] = endpoint; + } + var currentEndpointIds = endpoints.Select(e => e.Id).ToHashSet(); var existingEndpointIds = _endpointTimers.Keys.ToHashSet(); @@ -144,7 +153,8 @@ private async Task RefreshEndpointsAsync(CancellationToken cancellationToken) if (_endpointTimers.TryRemove(endpointId, out Timer? timer)) { await StopTimerGracefullyAsync(timer, endpointId); - _probeExecuting.TryRemove(endpointId, out _); // Clean up execution tracking + _probeExecuting.TryRemove(endpointId, out _); + _endpointCache.TryRemove(endpointId, out _); _logger.LogInformation("Stopped monitoring endpoint: {EndpointId}", endpointId); } } @@ -156,28 +166,36 @@ private async Task RefreshEndpointsAsync(CancellationToken cancellationToken) if (_endpointTimers.TryGetValue(endpoint.Id, out Timer? existingTimer)) { - // Stop timer to prevent race condition, then restart with new interval + // Only restart the timer if the interval changed — avoids thundering herd every refresh cycle + if (_endpointIntervals.TryGetValue(endpoint.Id, out int previousIntervalMs) && previousIntervalMs == intervalMs) + { + continue; + } + + // Interval changed: stop, wait for any in-flight probe, then restart existingTimer.Change(Timeout.Infinite, Timeout.Infinite); - // Wait briefly if probe is currently executing to avoid immediate restart if (_probeExecuting.TryGetValue(endpoint.Id, out bool isExecuting) && isExecuting) { - await Task.Delay(100); // Brief delay to let current execution complete + await Task.Delay(100); } - // Restart with new interval existingTimer.Change(TimeSpan.Zero, TimeSpan.FromMilliseconds(intervalMs)); + _endpointIntervals[endpoint.Id] = intervalMs; + _logger.LogInformation("Updated monitoring interval for endpoint: {EndpointId} ({Name}) to {IntervalSeconds}s", + endpoint.Id, endpoint.Name, endpoint.IntervalSeconds); } else { - // Create new timer for new endpoint + // New endpoint — start immediately var timer = new Timer( callback: async _ => await ProbeEndpointAsync(endpoint.Id), state: null, - dueTime: TimeSpan.Zero, // Start immediately + dueTime: TimeSpan.Zero, period: TimeSpan.FromMilliseconds(intervalMs)); _endpointTimers.TryAdd(endpoint.Id, timer); + _endpointIntervals.TryAdd(endpoint.Id, intervalMs); _logger.LogInformation("Started monitoring endpoint: {EndpointId} ({Name}) every {IntervalSeconds}s", endpoint.Id, endpoint.Name, endpoint.IntervalSeconds); } @@ -200,18 +218,16 @@ private async Task ProbeEndpointAsync(Guid endpointId) try { + // Use cached endpoint data — RefreshEndpointsAsync keeps this up to date every 15s + if (!_endpointCache.TryGetValue(endpointId, out Data.Endpoint? endpoint) || !endpoint.Enabled) + { + return; + } + using IServiceScope scope = _serviceProvider.CreateScope(); - PulseDbContext context = scope.ServiceProvider.GetRequiredService(); IProbeService probeService = scope.ServiceProvider.GetRequiredService(); IOutageDetectionService outageService = scope.ServiceProvider.GetRequiredService(); - // Get endpoint details - Data.Endpoint? endpoint = await context.Endpoints.FindAsync(endpointId); - if (endpoint == null || !endpoint.Enabled) - { - return; // Endpoint was deleted or disabled - } - // Perform the probe Models.CheckResult result = await probeService.ProbeAsync(endpoint); diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs b/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs index d2d6a67..0921478 100644 --- a/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs +++ b/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs @@ -14,12 +14,14 @@ public sealed class OutageDetectionService : IOutageDetectionService { private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; + private readonly ICheckResultWriteQueue _writeQueue; private readonly ConcurrentDictionary _states = new(); - public OutageDetectionService(IServiceProvider serviceProvider, ILogger logger) + public OutageDetectionService(IServiceProvider serviceProvider, ILogger logger, ICheckResultWriteQueue writeQueue) { _serviceProvider = serviceProvider; _logger = logger; + _writeQueue = writeQueue; } public async Task ProcessCheckResultAsync(CheckResult result, CancellationToken cancellationToken = default) @@ -69,8 +71,8 @@ public async Task ProcessCheckResultAsync(CheckResult result, Cancellation result.EndpointId, state.SuccessStreak); } - // Save the raw check result to database (independent of transitions) - await SaveCheckResultAsync(result, cancellationToken); + // Enqueue raw check result — written by the background write queue (non-blocking) + SaveCheckResult(result); return stateChanged; } @@ -515,11 +517,8 @@ private async Task UpdateEndpointStatusAsync(PulseDbContext context, Guid endpoi return (endpointStatus, openOutageId, false); } - private async Task SaveCheckResultAsync(CheckResult result, CancellationToken cancellationToken) + private void SaveCheckResult(CheckResult result) { - using IServiceScope scope = _serviceProvider.CreateScope(); - PulseDbContext context = scope.ServiceProvider.GetRequiredService(); - CheckResultRaw rawResult = new CheckResultRaw { EndpointId = result.EndpointId, @@ -529,21 +528,10 @@ private async Task SaveCheckResultAsync(CheckResult result, CancellationToken ca Error = result.Error }; - context.CheckResultsRaw.Add(rawResult); + Guid? endpointIdForRtt = result.Status == UpDown.up && result.RttMs.HasValue + ? result.EndpointId + : null; - // Update LastRttMs for successful probes - if (result.Status == UpDown.up && result.RttMs.HasValue) - { - Data.Endpoint? endpoint = await context.Endpoints.FindAsync([result.EndpointId], cancellationToken); - if (endpoint != null) - { - endpoint.LastRttMs = result.RttMs.Value; - _logger.LogInformation( - "Updated LastRttMs for endpoint {EndpointId} to {RttMs}ms", - result.EndpointId, result.RttMs.Value); - } - } - - await context.SaveChangesAsync(cancellationToken); + _writeQueue.Enqueue(new WriteQueueItem(rawResult, endpointIdForRtt, result.RttMs)); } } diff --git a/ThingConnect.Pulse.Server/Services/Prune/PruneBackgroundService.cs b/ThingConnect.Pulse.Server/Services/Prune/PruneBackgroundService.cs new file mode 100644 index 0000000..07d83c7 --- /dev/null +++ b/ThingConnect.Pulse.Server/Services/Prune/PruneBackgroundService.cs @@ -0,0 +1,90 @@ +namespace ThingConnect.Pulse.Server.Services.Prune; + +public sealed class PruneBackgroundService : BackgroundService +{ + private readonly IServiceProvider _serviceProvider; + private readonly IConfiguration _configuration; + private readonly ILogger _logger; + + public PruneBackgroundService( + IServiceProvider serviceProvider, + IConfiguration configuration, + ILogger logger) + { + _serviceProvider = serviceProvider; + _configuration = configuration; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Wait until the first 02:00 local time before running, then repeat every intervalHours + int intervalHours = _configuration.GetValue("Data:Pruning:IntervalHours", 24); + bool enabled = _configuration.GetValue("Data:Pruning:Enabled", true); + + if (!enabled) + { + _logger.LogInformation("Pruning is disabled via configuration"); + return; + } + + // Initial delay: time until next 02:00 local + TimeSpan initialDelay = TimeUntilNextRun(TimeSpan.FromHours(2)); + _logger.LogInformation( + "Prune background service scheduled. First run in {Delay:hh\\:mm\\:ss}, then every {IntervalHours}h", + initialDelay, intervalHours); + + try + { + await Task.Delay(initialDelay, stoppingToken); + } + catch (OperationCanceledException) + { + return; + } + + while (!stoppingToken.IsCancellationRequested) + { + await RunPruneAsync(stoppingToken); + + try + { + await Task.Delay(TimeSpan.FromHours(intervalHours), stoppingToken); + } + catch (OperationCanceledException) + { + break; + } + } + } + + private async Task RunPruneAsync(CancellationToken ct) + { + _logger.LogInformation("Starting scheduled raw data prune"); + var sw = System.Diagnostics.Stopwatch.StartNew(); + + try + { + using IServiceScope scope = _serviceProvider.CreateScope(); + IPruneService pruneService = scope.ServiceProvider.GetRequiredService(); + + int deleted = await pruneService.PruneRawDataAsync(dryRun: false, ct); + + _logger.LogInformation( + "Scheduled prune complete: deleted {Deleted} raw check results in {Elapsed}ms", + deleted, sw.ElapsedMilliseconds); + } + catch (Exception ex) + { + _logger.LogError(ex, "Scheduled prune failed after {Elapsed}ms", sw.ElapsedMilliseconds); + } + } + + private static TimeSpan TimeUntilNextRun(TimeSpan targetTimeOfDay) + { + DateTimeOffset now = DateTimeOffset.Now; + DateTimeOffset next = now.Date + targetTimeOfDay; + if (next <= now) next = next.AddDays(1); + return next - now; + } +} diff --git a/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs b/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs index c49e9b1..b66fa70 100644 --- a/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs +++ b/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs @@ -33,14 +33,13 @@ public async Task ProcessRollup15mAsync(CancellationToken cancellationToken = de _logger.LogDebug("Processing 15m rollups from {FromTs} to {ToTs}", UnixTimestamp.FromUnixSeconds(fromTs), UnixTimestamp.FromUnixSeconds(toTs)); - // Get all raw checks in the time window - // SQLite has issues with DateTimeOffset comparisons in LINQ, so fetch all and filter in memory - List allChecks = await _context.CheckResultsRaw.ToListAsync(cancellationToken); - var rawChecks = allChecks + // Get raw checks in the time window — Ts is a long (unix seconds), safe to filter in SQL + List rawChecks = await _context.CheckResultsRaw .Where(c => c.Ts > fromTs && c.Ts <= toTs) + .AsNoTracking() .OrderBy(c => c.EndpointId) .ThenBy(c => c.Ts) - .ToList(); + .ToListAsync(cancellationToken); if (!rawChecks.Any()) { @@ -95,17 +94,16 @@ public async Task ProcessRollupDailyAsync(CancellationToken cancellationToken = _logger.LogDebug("Processing daily rollups from {FromDate} to {ToDate}", fromDate, toDate); - // Get all raw checks in the date range + // Get raw checks in the date range — Ts is a long (unix seconds), safe to filter in SQL long fromTs = UnixTimestamp.ToUnixDate(fromDate); long toTs = UnixTimestamp.ToUnixDate(toDate); - // SQLite has issues with DateTimeOffset comparisons in LINQ, so fetch all and filter in memory - List allChecks = await _context.CheckResultsRaw.ToListAsync(cancellationToken); - var rawChecks = allChecks + List rawChecks = await _context.CheckResultsRaw .Where(c => c.Ts >= fromTs && c.Ts < toTs) + .AsNoTracking() .OrderBy(c => c.EndpointId) .ThenBy(c => c.Ts) - .ToList(); + .ToListAsync(cancellationToken); if (!rawChecks.Any()) { diff --git a/ThingConnect.Pulse.Server/Services/StatusService.cs b/ThingConnect.Pulse.Server/Services/StatusService.cs index 3fade8c..9e64cf6 100644 --- a/ThingConnect.Pulse.Server/Services/StatusService.cs +++ b/ThingConnect.Pulse.Server/Services/StatusService.cs @@ -50,46 +50,35 @@ public async Task> GetLiveStatusAsync(string? group, str e.Host.ToLower().Contains(searchLower)); } - // Get total count for pagination - int totalCount = await query.CountAsync(); - - // Apply pagination List endpoints = await query - .OrderBy(e => e.GroupId) - .ThenBy(e => e.Name) - .ToListAsync(); + .OrderBy(e => e.GroupId) + .ThenBy(e => e.Name) + .ToListAsync(); - // Get live status for each endpoint var items = new List(); var endpointIds = endpoints.Select(e => e.Id).ToList(); - // Get latest checks for all endpoints - optimized query using window functions in SQLite - var latestChecks = await _context.CheckResultsRaw - .Where(c => endpointIds.Contains(c.EndpointId)) - .AsNoTracking() - .GroupBy(c => c.EndpointId) - .Select(g => new - { - EndpointId = g.Key, - LatestCheck = g.OrderByDescending(c => c.Ts).FirstOrDefault() - }) - .ToListAsync(); - - var latestCheckDict = latestChecks.ToDictionary(x => x.EndpointId, x => x.LatestCheck); - - // Get sparkline data (last 20 checks per endpoint for mini chart) + // Sparkline query (2h window) — last point also serves as latest check timestamp Dictionary> sparklineData = await GetSparklineDataAsync(endpointIds); + // Single 5min query for flap detection + HashSet flappingIds = await GetFlappingEndpointIdsAsync(endpointIds); + foreach (Data.Endpoint? endpoint in endpoints) { - StatusType status = DetermineStatus(endpoint, latestCheckDict); - List sparkline = sparklineData.ContainsKey(endpoint.Id) - ? sparklineData[endpoint.Id] - : new List(); + List sparkline = sparklineData.GetValueOrDefault(endpoint.Id, new List()); + + // Most recent sparkline point gives us the last check time — no extra check_result_raw query needed + long? latestCheckTs = sparkline.Count > 0 + ? UnixTimestamp.ToUnixSeconds(sparkline.Last().Ts) + : null; - _logger.LogInformation( - "Endpoint {EndpointName}: Status = {Status}, LastRttMs = {RttMs}, LastChangeTs = {LastChangeTs}", - endpoint.Name, status, endpoint.LastRttMs, endpoint.LastChangeTs); + // endpoint.LastStatus/LastRttMs/LastChangeTs are maintained by OutageDetectionService + StatusType status = DetermineStatus(endpoint, latestCheckTs, flappingIds); + + _logger.LogDebug( + "Endpoint {EndpointName}: Status={Status}, RttMs={RttMs}", + endpoint.Name, status, endpoint.LastRttMs); items.Add(new LiveStatusItemDto { @@ -180,61 +169,54 @@ private async Task>> GetSparklineDataAsync return sparklineData; } - private StatusType DetermineStatus(Data.Endpoint endpoint, Dictionary latestChecks) + private StatusType DetermineStatus(Data.Endpoint endpoint, long? latestCheckTs, HashSet flappingIds) { - // Check if we have recent check data - if (!latestChecks.TryGetValue(endpoint.Id, out CheckResultRaw? latestCheck) || latestCheck == null) + if (latestCheckTs == null) { - return StatusType.Down; // No data means down + return StatusType.Down; } - // Check if the latest check is recent enough (within 2x interval) - var expectedInterval = TimeSpan.FromSeconds(endpoint.IntervalSeconds * 2); - if (UnixTimestamp.Now() - latestCheck.Ts > (long)expectedInterval.TotalSeconds) + if (UnixTimestamp.Now() - latestCheckTs.Value > endpoint.IntervalSeconds * 2) { - return StatusType.Down; // Stale data means down + return StatusType.Down; } - // Check for flapping (multiple state changes in short period) - // This is simplified - in production you'd want more sophisticated flap detection - if (IsFlapping(endpoint.Id).Result) + if (flappingIds.Contains(endpoint.Id)) { return StatusType.Flapping; } - return latestCheck.Status == UpDown.up ? StatusType.Up : StatusType.Down; + return endpoint.LastStatus == UpDown.up ? StatusType.Up : StatusType.Down; } - private async Task IsFlapping(Guid endpointId) + // Single query for all endpoints instead of one query per endpoint with .Result (which causes thread pool starvation) + private async Task> GetFlappingEndpointIdsAsync(List endpointIds) { - // Simple flap detection: check if there were > 3 state changes in last 5 minutes long cutoffTime = UnixTimestamp.Subtract(UnixTimestamp.Now(), TimeSpan.FromMinutes(5)); - var checks = await _context.CheckResultsRaw - .Where(c => c.EndpointId == endpointId && c.Ts >= cutoffTime) + + var recentChecks = await _context.CheckResultsRaw + .Where(c => endpointIds.Contains(c.EndpointId) && c.Ts >= cutoffTime) .AsNoTracking() - .Select(c => new { c.Ts, c.Status }) + .Select(c => new { c.EndpointId, c.Ts, c.Status }) .ToListAsync(); - var recentChecks = checks - .OrderBy(c => c.Ts) - .Select(c => c.Status) - .ToList(); + var flapping = new HashSet(); - if (recentChecks.Count < 4) + foreach (var group in recentChecks.GroupBy(c => c.EndpointId)) { - return false; - } + var statuses = group.OrderBy(c => c.Ts).Select(c => c.Status).ToList(); + if (statuses.Count < 4) continue; - int stateChanges = 0; - for (int i = 1; i < recentChecks.Count; i++) - { - if (recentChecks[i] != recentChecks[i - 1]) + int changes = 0; + for (int i = 1; i < statuses.Count; i++) { - stateChanges++; + if (statuses[i] != statuses[i - 1]) changes++; } + + if (changes > 3) flapping.Add(group.Key); } - return stateChanges > 3; + return flapping; } private EndpointDto MapToEndpointDto(Data.Endpoint endpoint) diff --git a/ThingConnect.Pulse.Server/appsettings.json b/ThingConnect.Pulse.Server/appsettings.json index 1913b32..69dc01c 100644 --- a/ThingConnect.Pulse.Server/appsettings.json +++ b/ThingConnect.Pulse.Server/appsettings.json @@ -1,13 +1,15 @@ { "Serilog": { - "Using": ["Serilog.Sinks.Console", "Serilog.Sinks.File"], + "Using": [ + "Serilog.Sinks.Console", + "Serilog.Sinks.File" + ], "MinimumLevel": { "Default": "Information", "Override": { "Microsoft": "Warning", "Microsoft.Hosting.Lifetime": "Information", "Microsoft.EntityFrameworkCore": "Warning", - "Microsoft.EntityFrameworkCore.Database.Command": "Debug", "System": "Warning" } }, @@ -31,7 +33,12 @@ } } ], - "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"], + "Enrich": [ + "FromLogContext", + "WithMachineName", + "WithProcessId", + "WithThreadId" + ], "Properties": { "Application": "ThingConnect.Pulse", "Environment": "Development" @@ -80,4 +87,4 @@ "Monitoring": { "MaxConcurrentProbes": 100 } -} +} \ No newline at end of file diff --git a/ThingConnect.Pulse.Tests/CidrExpansionTests.cs b/ThingConnect.Pulse.Tests/CidrExpansionTests.cs index d16c77a..abe18f8 100644 --- a/ThingConnect.Pulse.Tests/CidrExpansionTests.cs +++ b/ThingConnect.Pulse.Tests/CidrExpansionTests.cs @@ -36,7 +36,8 @@ private IEnumerable ExpandCidr(string cidr) } byte[] addressBytes = ipAddress.GetAddressBytes(); - uint addressInt = BitConverter.ToUInt32(addressBytes.Reverse().ToArray(), 0); + Array.Reverse(addressBytes); + uint addressInt = BitConverter.ToUInt32(addressBytes, 0); int hostBits = 32 - prefixLength; uint hostCount = (uint)(1 << hostBits); @@ -48,7 +49,8 @@ private IEnumerable ExpandCidr(string cidr) for (uint address = startAddress; address < endAddress && address > networkAddress; address++) { - byte[] bytes = BitConverter.GetBytes(address).Reverse().ToArray(); + byte[] bytes = BitConverter.GetBytes(address); + Array.Reverse(bytes); var ip = new IPAddress(bytes); yield return ip.ToString(); } diff --git a/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs b/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs index 5941efd..05b6748 100644 --- a/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs +++ b/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs @@ -151,7 +151,8 @@ private List ExpandCidrForTesting(string cidr) } byte[] addressBytes = ipAddress.GetAddressBytes(); - uint addressInt = BitConverter.ToUInt32(addressBytes.Reverse().ToArray(), 0); + Array.Reverse(addressBytes); + uint addressInt = BitConverter.ToUInt32(addressBytes, 0); int hostBits = 32 - prefixLength; uint hostCount = (uint)(1 << hostBits); @@ -163,7 +164,8 @@ private List ExpandCidrForTesting(string cidr) for (uint address = startAddress; address < endAddress && address > networkAddress; address++) { - byte[] bytes = BitConverter.GetBytes(address).Reverse().ToArray(); + byte[] bytes = BitConverter.GetBytes(address); + Array.Reverse(bytes); var ip = new System.Net.IPAddress(bytes); result.Add(ip.ToString()); } diff --git a/docs/fix-ram-usage.md b/docs/fix-ram-usage.md new file mode 100644 index 0000000..e15e5b3 --- /dev/null +++ b/docs/fix-ram-usage.md @@ -0,0 +1,91 @@ +# Fix: High RAM Usage (~70% on Windows Service) + +## Root Causes + +### 1. Thundering herd in MonitoringBackgroundService ✅ Fixed +**File:** `ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs` + +Every 15 seconds the refresh loop restarted **all** endpoint timers with `dueTime: TimeSpan.Zero`, firing every probe simultaneously regardless of configured intervals. With 200 concurrent probes, this caused bursts of 200 EF Core DbContext instances + tasks every 15 seconds. + +**Fix applied:** Timers are now only restarted when the interval changes. Existing timers run undisturbed on their own schedule. + +--- + +### 2. MaxConcurrentProbes too high ✅ Fixed +**File:** `ThingConnect.Pulse.Server/appsettings.json` + +`MaxConcurrentProbes` was set to 200. Each concurrent probe creates a DbContext (~50–100KB each) plus async task overhead. + +**Fix applied:** Lowered to 100. + +--- + +### 3. EF Core SQL debug logging ✅ Fixed +**File:** `ThingConnect.Pulse.Server/appsettings.json` + +`"Microsoft.EntityFrameworkCore.Database.Command": "Debug"` was logging every SQL query. With hundreds of probes per minute, Serilog's internal buffer accumulated thousands of log strings per minute. + +**Fix applied:** Removed the debug override, leaving EF Core at `Warning`. + +--- + +### 4. RollupService loads entire check_results_raw table every 5 minutes ❌ Not fixed yet +**File:** `ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs` — Lines 38, 103 + +```csharp +// PROBLEM: loads every row in the table into memory +List allChecks = await _context.CheckResultsRaw.ToListAsync(cancellationToken); +var rawChecks = allChecks.Where(c => c.Ts > fromTs && c.Ts <= toTs)... +``` + +The comment says "SQLite has issues with DateTimeOffset comparisons" but `Ts` is stored as a `long` unix timestamp — EF Core pushes `long` comparisons to SQL without issue. With 60-day retention at moderate probe frequency this is potentially millions of rows loaded every 5 minutes. + +**Fix needed:** +```csharp +List rawChecks = await _context.CheckResultsRaw + .Where(c => c.Ts > fromTs && c.Ts <= toTs) + .AsNoTracking() + .OrderBy(c => c.EndpointId).ThenBy(c => c.Ts) + .ToListAsync(cancellationToken); +``` +Apply the same change to both `ProcessRollup15mAsync` (line 38) and `ProcessRollupDailyAsync` (line 103). + +--- + +### 5. HistoryService loads all rows for an endpoint then filters in memory ❌ Not fixed yet +**File:** `ThingConnect.Pulse.Server/Services/HistoryService.cs` — Lines 88, 112, 137, 161 + +All four query methods (`GetRawDataAsync`, `GetRollup15mDataAsync`, `GetRollupDailyDataAsync`, `GetOutagesAsync`) follow the same pattern: + +```csharp +// PROBLEM: loads all rows for the endpoint, filters by time in C# +var data = await _context.CheckResultsRaw + .Where(c => c.EndpointId == endpointId) + .ToListAsync(); +return data.Where(c => c.Ts >= fromUnix && c.Ts <= toUnix)... +``` + +**Fix needed:** Push the time filter into SQL and add `AsNoTracking()`: +```csharp +return await _context.CheckResultsRaw + .Where(c => c.EndpointId == endpointId && c.Ts >= fromUnix && c.Ts <= toUnix) + .AsNoTracking() + .OrderBy(c => c.Ts) + .Select(c => new RawCheckDto { ... }) + .ToListAsync(); +``` +Apply to all four methods — same pattern, just add the timestamp condition into the existing `.Where()`. + +--- + +## Priority + +| # | Issue | Impact | Status | +|---|-------|--------|--------| +| 1 | Thundering herd (timer restart) | High | ✅ Fixed | +| 2 | MaxConcurrentProbes = 200 | Medium | ✅ Fixed | +| 3 | EF Core debug logging | Medium | ✅ Fixed | +| 4 | RollupService full table scan | **Critical** | ✅ Fixed | +| 5 | HistoryService full scan per request | High | ✅ Fixed | + +All five issues resolved. diff --git a/setup.iss b/setup.iss index 24ac026..7ac814a 100644 --- a/setup.iss +++ b/setup.iss @@ -23,7 +23,7 @@ DefaultDirName={autopf}\ThingConnect.Pulse DefaultGroupName={#AppName} DisableProgramGroupPage=yes OutputDir=installer -OutputBaseFilename=ThingConnect Pulse - Setup {#AppVersion} +OutputBaseFilename=ThingConnect.Pulse.Setup {#AppVersion} Compression=lzma SolidCompression=yes WizardStyle=modern diff --git a/thingconnect.pulse.client/vite.config.ts b/thingconnect.pulse.client/vite.config.ts index 62f79ec..452b44f 100644 --- a/thingconnect.pulse.client/vite.config.ts +++ b/thingconnect.pulse.client/vite.config.ts @@ -38,7 +38,7 @@ const target = env.ASPNETCORE_HTTPS_PORT ? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}` : env.ASPNETCORE_URLS ? env.ASPNETCORE_URLS.split(';')[0] - : 'http://localhost:8080'; + : 'http://localhost:8090'; // https://vitejs.dev/config/ export default defineConfig({ @@ -49,10 +49,12 @@ export default defineConfig({ }, }, server: { + host: '0.0.0.0', proxy: { '^/api': { target, secure: false, + changeOrigin: true, }, }, port: parseInt(env.DEV_SERVER_PORT || '55605'),