diff --git a/ThingConnect.Pulse.Server/Infrastructure/SqliteWalInterceptor.cs b/ThingConnect.Pulse.Server/Infrastructure/SqliteWalInterceptor.cs
new file mode 100644
index 0000000..2737545
--- /dev/null
+++ b/ThingConnect.Pulse.Server/Infrastructure/SqliteWalInterceptor.cs
@@ -0,0 +1,51 @@
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using System.Data.Common;
+
+namespace ThingConnect.Pulse.Server.Infrastructure;
+
+///
+/// Enables WAL mode and busy_timeout on every SQLite connection to allow concurrent reads
+/// during high-frequency writes from the monitoring background service.
+///
+/// Without WAL, SQLite's default DELETE journal mode holds an exclusive lock for the
+/// duration of each write, causing read timeouts when probes are writing continuously.
+/// WAL mode decouples readers from writers entirely.
+///
+public sealed class SqliteWalInterceptor : DbConnectionInterceptor
+{
+ public override void ConnectionOpened(DbConnection connection, ConnectionEndEventData eventData)
+ {
+ ApplyPragmas(connection);
+ }
+
+ public override async Task ConnectionOpenedAsync(
+ DbConnection connection,
+ ConnectionEndEventData eventData,
+ CancellationToken cancellationToken = default)
+ {
+ await ApplyPragmasAsync(connection, cancellationToken);
+ }
+
+ [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100", Justification = "Pragma SQL is a compile-time constant, not user input.")]
+ private static void ApplyPragmas(DbConnection connection)
+ {
+ using DbCommand cmd = connection.CreateCommand();
+ cmd.CommandText = BuildPragmaSql();
+ cmd.ExecuteNonQuery();
+ }
+
+ [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA2100", Justification = "Pragma SQL is a compile-time constant, not user input.")]
+ private static async Task ApplyPragmasAsync(DbConnection connection, CancellationToken ct)
+ {
+ await using DbCommand cmd = connection.CreateCommand();
+ cmd.CommandText = BuildPragmaSql();
+ await cmd.ExecuteNonQueryAsync(ct);
+ }
+
+ private static string BuildPragmaSql() => """
+ PRAGMA journal_mode=WAL;
+ PRAGMA busy_timeout=5000;
+ PRAGMA synchronous=NORMAL;
+ PRAGMA cache_size=-8000;
+ """;
+}
diff --git a/ThingConnect.Pulse.Server/Program.cs b/ThingConnect.Pulse.Server/Program.cs
index 75aac7d..95873c2 100644
--- a/ThingConnect.Pulse.Server/Program.cs
+++ b/ThingConnect.Pulse.Server/Program.cs
@@ -48,7 +48,8 @@ public static async Task Main(string[] args)
// Add services to the container.
builder.Services.AddDbContext(options =>
- options.UseSqlite(builder.Configuration.GetConnectionString("DefaultConnection")));
+ options.UseSqlite(builder.Configuration.GetConnectionString("DefaultConnection"))
+ .AddInterceptors(new SqliteWalInterceptor()));
// Configure Identity and Authentication
builder.Services.AddIdentity(options =>
@@ -86,7 +87,7 @@ public static async Task Main(string[] args)
options.Cookie.HttpOnly = true;
options.Cookie.SecurePolicy = builder.Environment.IsDevelopment()
? CookieSecurePolicy.SameAsRequest
- : CookieSecurePolicy.Always;
+ : CookieSecurePolicy.None;
options.Cookie.SameSite = SameSiteMode.Lax;
options.Cookie.Name = "ThingConnect.Pulse.Auth";
options.Events.OnRedirectToLogin = context =>
@@ -150,6 +151,9 @@ public static async Task Main(string[] args)
builder.Services.AddSingleton();
// Add monitoring services
+ builder.Services.AddSingleton();
+ builder.Services.AddSingleton(p => p.GetRequiredService());
+ builder.Services.AddHostedService(p => p.GetRequiredService());
builder.Services.AddScoped();
builder.Services.AddSingleton();
builder.Services.AddSingleton();
@@ -164,6 +168,7 @@ public static async Task Main(string[] args)
// Add prune services
builder.Services.AddScoped();
+ builder.Services.AddHostedService();
// Add log cleanup service
builder.Services.AddHostedService();
diff --git a/ThingConnect.Pulse.Server/Services/HistoryService.cs b/ThingConnect.Pulse.Server/Services/HistoryService.cs
index 6c1c224..86b4eda 100644
--- a/ThingConnect.Pulse.Server/Services/HistoryService.cs
+++ b/ThingConnect.Pulse.Server/Services/HistoryService.cs
@@ -84,14 +84,9 @@ private async Task> GetRawDataAsync(Guid endpointId, DateTimeO
long fromUnix = UnixTimestamp.ToUnixSeconds(from);
long toUnix = UnixTimestamp.ToUnixSeconds(to);
- // SQLite limitation: fetch all data and filter in memory
- var rawData = await _context.CheckResultsRaw
- .Where(c => c.EndpointId == endpointId)
- .Select(c => new { c.Ts, c.Status, c.RttMs, c.Error })
- .ToListAsync();
-
- return rawData
- .Where(c => c.Ts >= fromUnix && c.Ts <= toUnix)
+ return await _context.CheckResultsRaw
+ .Where(c => c.EndpointId == endpointId && c.Ts >= fromUnix && c.Ts <= toUnix)
+ .AsNoTracking()
.OrderBy(c => c.Ts)
.Select(c => new RawCheckDto
{
@@ -100,7 +95,7 @@ private async Task> GetRawDataAsync(Guid endpointId, DateTimeO
RttMs = c.RttMs,
Error = c.Error
})
- .ToList();
+ .ToListAsync();
}
private async Task> GetRollup15mDataAsync(Guid endpointId, DateTimeOffset from, DateTimeOffset to)
@@ -108,14 +103,9 @@ private async Task> GetRollup15mDataAsync(Guid endpointId,
long fromUnix = UnixTimestamp.ToUnixSeconds(from);
long toUnix = UnixTimestamp.ToUnixSeconds(to);
- // SQLite limitation: fetch all data and filter in memory
- var rollupData = await _context.Rollups15m
- .Where(r => r.EndpointId == endpointId)
- .Select(r => new { r.BucketTs, r.UpPct, r.AvgRttMs, r.DownEvents })
- .ToListAsync();
-
- return rollupData
- .Where(r => r.BucketTs >= fromUnix && r.BucketTs <= toUnix)
+ return await _context.Rollups15m
+ .Where(r => r.EndpointId == endpointId && r.BucketTs >= fromUnix && r.BucketTs <= toUnix)
+ .AsNoTracking()
.OrderBy(r => r.BucketTs)
.Select(r => new RollupBucketDto
{
@@ -124,7 +114,7 @@ private async Task> GetRollup15mDataAsync(Guid endpointId,
AvgRttMs = r.AvgRttMs,
DownEvents = r.DownEvents
})
- .ToList();
+ .ToListAsync();
}
private async Task> GetRollupDailyDataAsync(Guid endpointId, DateTimeOffset from, DateTimeOffset to)
@@ -133,14 +123,9 @@ private async Task> GetRollupDailyDataAsync(Guid endpointId
long fromUnix = UnixTimestamp.ToUnixDate(DateOnly.FromDateTime(from.Date));
long toUnix = UnixTimestamp.ToUnixDate(DateOnly.FromDateTime(to.Date));
- // SQLite limitation: fetch all data and filter in memory
- var dailyData = await _context.RollupsDaily
- .Where(r => r.EndpointId == endpointId)
- .Select(r => new { r.BucketDate, r.UpPct, r.AvgRttMs, r.DownEvents })
- .ToListAsync();
-
- return dailyData
- .Where(r => r.BucketDate >= fromUnix && r.BucketDate <= toUnix)
+ return await _context.RollupsDaily
+ .Where(r => r.EndpointId == endpointId && r.BucketDate >= fromUnix && r.BucketDate <= toUnix)
+ .AsNoTracking()
.OrderBy(r => r.BucketDate)
.Select(r => new DailyBucketDto
{
@@ -149,7 +134,7 @@ private async Task> GetRollupDailyDataAsync(Guid endpointId
AvgRttMs = r.AvgRttMs,
DownEvents = r.DownEvents
})
- .ToList();
+ .ToListAsync();
}
private async Task> GetOutagesAsync(Guid endpointId, DateTimeOffset from, DateTimeOffset to)
@@ -157,14 +142,9 @@ private async Task> GetOutagesAsync(Guid endpointId, DateTimeOff
long fromUnix = UnixTimestamp.ToUnixSeconds(from);
long toUnix = UnixTimestamp.ToUnixSeconds(to);
- // SQLite limitation: fetch all data and filter in memory
- var outageData = await _context.Outages
- .Where(o => o.EndpointId == endpointId)
- .Select(o => new { o.StartedTs, o.EndedTs, o.DurationSeconds, o.LastError })
- .ToListAsync();
-
- return outageData
- .Where(o => o.StartedTs <= toUnix && (o.EndedTs == null || o.EndedTs >= fromUnix))
+ return await _context.Outages
+ .Where(o => o.EndpointId == endpointId && o.StartedTs <= toUnix && (o.EndedTs == null || o.EndedTs >= fromUnix))
+ .AsNoTracking()
.OrderBy(o => o.StartedTs)
.Select(o => new OutageDto
{
@@ -173,7 +153,7 @@ private async Task> GetOutagesAsync(Guid endpointId, DateTimeOff
DurationS = o.DurationSeconds,
LastError = o.LastError
})
- .ToList();
+ .ToListAsync();
}
private EndpointDto MapToEndpointDto(Data.Endpoint endpoint)
diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/CheckResultWriteQueue.cs b/ThingConnect.Pulse.Server/Services/Monitoring/CheckResultWriteQueue.cs
new file mode 100644
index 0000000..62aa801
--- /dev/null
+++ b/ThingConnect.Pulse.Server/Services/Monitoring/CheckResultWriteQueue.cs
@@ -0,0 +1,122 @@
+using System.Threading.Channels;
+using Microsoft.EntityFrameworkCore;
+using ThingConnect.Pulse.Server.Data;
+
+namespace ThingConnect.Pulse.Server.Services.Monitoring;
+
+public interface ICheckResultWriteQueue
+{
+ void Enqueue(WriteQueueItem item);
+}
+
+public sealed record WriteQueueItem(
+ CheckResultRaw RawResult,
+ Guid? EndpointIdForRttUpdate,
+ double? RttMs);
+
+///
+/// Serializes all check_result_raw inserts through a single background writer, eliminating
+/// the concurrent SaveChangesAsync flood that causes SQLite read timeouts under high probe rates.
+///
+public sealed class CheckResultWriteQueue : BackgroundService, ICheckResultWriteQueue
+{
+ private readonly Channel _channel = Channel.CreateBounded(
+ new BoundedChannelOptions(10_000) { FullMode = BoundedChannelFullMode.DropOldest });
+
+ private readonly IServiceProvider _serviceProvider;
+ private readonly ILogger _logger;
+
+ public CheckResultWriteQueue(IServiceProvider serviceProvider, ILogger logger)
+ {
+ _serviceProvider = serviceProvider;
+ _logger = logger;
+ }
+
+ public void Enqueue(WriteQueueItem item) => _channel.Writer.TryWrite(item);
+
+ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+ {
+ _logger.LogInformation("Check result write queue started");
+
+ var batch = new List(100);
+
+ while (!stoppingToken.IsCancellationRequested)
+ {
+ batch.Clear();
+
+ try
+ {
+ // Block until at least one item is available
+ if (!await _channel.Reader.WaitToReadAsync(stoppingToken))
+ {
+ break;
+ }
+
+ // Drain all ready items up to batch limit (no extra waiting)
+ while (batch.Count < 100 && _channel.Reader.TryRead(out WriteQueueItem? item))
+ {
+ batch.Add(item);
+ }
+
+ await FlushBatchAsync(batch, stoppingToken);
+ }
+ catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+ {
+ break;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Error flushing check result batch of {Count} items", batch.Count);
+ // Brief pause to avoid tight error loop, then continue
+ await Task.Delay(1000, stoppingToken).ConfigureAwait(false);
+ }
+ }
+
+ // Drain remaining items on shutdown
+ batch.Clear();
+ while (_channel.Reader.TryRead(out WriteQueueItem? item))
+ {
+ batch.Add(item);
+ }
+
+ if (batch.Count > 0)
+ {
+ try
+ {
+ await FlushBatchAsync(batch, CancellationToken.None);
+ _logger.LogInformation("Flushed {Count} remaining check results on shutdown", batch.Count);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Failed to flush remaining {Count} check results on shutdown", batch.Count);
+ }
+ }
+
+ _logger.LogInformation("Check result write queue stopped");
+ }
+
+ private async Task FlushBatchAsync(List batch, CancellationToken ct)
+ {
+ using IServiceScope scope = _serviceProvider.CreateScope();
+ PulseDbContext context = scope.ServiceProvider.GetRequiredService();
+
+ // Bulk insert all check results in one round-trip
+ context.CheckResultsRaw.AddRange(batch.Select(i => i.RawResult));
+ await context.SaveChangesAsync(ct);
+
+ // Update LastRttMs with direct UPDATE statements — no entity load needed
+ var rttUpdates = batch
+ .Where(i => i.EndpointIdForRttUpdate.HasValue && i.RttMs.HasValue)
+ .GroupBy(i => i.EndpointIdForRttUpdate!.Value)
+ .Select(g => (EndpointId: g.Key, RttMs: g.Last().RttMs!.Value));
+
+ foreach ((Guid endpointId, double rttMs) in rttUpdates)
+ {
+ await context.Endpoints
+ .Where(e => e.Id == endpointId)
+ .ExecuteUpdateAsync(s => s.SetProperty(e => e.LastRttMs, rttMs), ct);
+ }
+
+ _logger.LogDebug("Flushed batch of {Count} check results", batch.Count);
+ }
+}
diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs b/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs
index 44a97fe..6b3d7e0 100644
--- a/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs
+++ b/ThingConnect.Pulse.Server/Services/Monitoring/DiscoveryService.cs
@@ -1,3 +1,4 @@
+using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text.RegularExpressions;
@@ -47,7 +48,7 @@ public IEnumerable ExpandCidr(string cidr)
}
byte[] addressBytes = ipAddress.GetAddressBytes();
- uint addressInt = BitConverter.ToUInt32(addressBytes.Reverse().ToArray(), 0);
+ uint addressInt = BitConverter.ToUInt32(addressBytes, 0);
int hostBits = 32 - prefixLength;
uint hostCount = (uint)(1 << hostBits);
@@ -59,7 +60,8 @@ public IEnumerable ExpandCidr(string cidr)
for (uint address = startAddress; address < endAddress && address > networkAddress; address++)
{
- byte[] bytes = BitConverter.GetBytes(address).Reverse().ToArray();
+ byte[] bytes = BitConverter.GetBytes(address);
+ System.Array.Reverse(bytes);
var ip = new IPAddress(bytes);
yield return ip.ToString();
}
diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs b/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs
index bced0e8..053b726 100644
--- a/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs
+++ b/ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs
@@ -15,6 +15,8 @@ public sealed class MonitoringBackgroundService : BackgroundService
private readonly SemaphoreSlim _concurrencySemaphore;
private readonly ConcurrentDictionary _endpointTimers = new();
private readonly ConcurrentDictionary _probeExecuting = new();
+ private readonly ConcurrentDictionary _endpointCache = new();
+ private readonly ConcurrentDictionary _endpointIntervals = new();
private readonly int _maxConcurrentProbes;
public MonitoringBackgroundService(IServiceProvider serviceProvider,
@@ -120,6 +122,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
}
_endpointTimers.Clear();
+ _endpointIntervals.Clear();
_logger.LogInformation("Monitoring background service stopped");
}
@@ -134,6 +137,12 @@ private async Task RefreshEndpointsAsync(CancellationToken cancellationToken)
.Where(e => e.Enabled)
.ToListAsync(cancellationToken);
+ // Refresh in-memory cache so probes don't need to re-read endpoint data from DB
+ foreach (Data.Endpoint endpoint in endpoints)
+ {
+ _endpointCache[endpoint.Id] = endpoint;
+ }
+
var currentEndpointIds = endpoints.Select(e => e.Id).ToHashSet();
var existingEndpointIds = _endpointTimers.Keys.ToHashSet();
@@ -144,7 +153,8 @@ private async Task RefreshEndpointsAsync(CancellationToken cancellationToken)
if (_endpointTimers.TryRemove(endpointId, out Timer? timer))
{
await StopTimerGracefullyAsync(timer, endpointId);
- _probeExecuting.TryRemove(endpointId, out _); // Clean up execution tracking
+ _probeExecuting.TryRemove(endpointId, out _);
+ _endpointCache.TryRemove(endpointId, out _);
_logger.LogInformation("Stopped monitoring endpoint: {EndpointId}", endpointId);
}
}
@@ -156,28 +166,36 @@ private async Task RefreshEndpointsAsync(CancellationToken cancellationToken)
if (_endpointTimers.TryGetValue(endpoint.Id, out Timer? existingTimer))
{
- // Stop timer to prevent race condition, then restart with new interval
+ // Only restart the timer if the interval changed — avoids thundering herd every refresh cycle
+ if (_endpointIntervals.TryGetValue(endpoint.Id, out int previousIntervalMs) && previousIntervalMs == intervalMs)
+ {
+ continue;
+ }
+
+ // Interval changed: stop, wait for any in-flight probe, then restart
existingTimer.Change(Timeout.Infinite, Timeout.Infinite);
- // Wait briefly if probe is currently executing to avoid immediate restart
if (_probeExecuting.TryGetValue(endpoint.Id, out bool isExecuting) && isExecuting)
{
- await Task.Delay(100); // Brief delay to let current execution complete
+ await Task.Delay(100);
}
- // Restart with new interval
existingTimer.Change(TimeSpan.Zero, TimeSpan.FromMilliseconds(intervalMs));
+ _endpointIntervals[endpoint.Id] = intervalMs;
+ _logger.LogInformation("Updated monitoring interval for endpoint: {EndpointId} ({Name}) to {IntervalSeconds}s",
+ endpoint.Id, endpoint.Name, endpoint.IntervalSeconds);
}
else
{
- // Create new timer for new endpoint
+ // New endpoint — start immediately
var timer = new Timer(
callback: async _ => await ProbeEndpointAsync(endpoint.Id),
state: null,
- dueTime: TimeSpan.Zero, // Start immediately
+ dueTime: TimeSpan.Zero,
period: TimeSpan.FromMilliseconds(intervalMs));
_endpointTimers.TryAdd(endpoint.Id, timer);
+ _endpointIntervals.TryAdd(endpoint.Id, intervalMs);
_logger.LogInformation("Started monitoring endpoint: {EndpointId} ({Name}) every {IntervalSeconds}s",
endpoint.Id, endpoint.Name, endpoint.IntervalSeconds);
}
@@ -200,18 +218,16 @@ private async Task ProbeEndpointAsync(Guid endpointId)
try
{
+ // Use cached endpoint data — RefreshEndpointsAsync keeps this up to date every 15s
+ if (!_endpointCache.TryGetValue(endpointId, out Data.Endpoint? endpoint) || !endpoint.Enabled)
+ {
+ return;
+ }
+
using IServiceScope scope = _serviceProvider.CreateScope();
- PulseDbContext context = scope.ServiceProvider.GetRequiredService();
IProbeService probeService = scope.ServiceProvider.GetRequiredService();
IOutageDetectionService outageService = scope.ServiceProvider.GetRequiredService();
- // Get endpoint details
- Data.Endpoint? endpoint = await context.Endpoints.FindAsync(endpointId);
- if (endpoint == null || !endpoint.Enabled)
- {
- return; // Endpoint was deleted or disabled
- }
-
// Perform the probe
Models.CheckResult result = await probeService.ProbeAsync(endpoint);
diff --git a/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs b/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs
index d2d6a67..0921478 100644
--- a/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs
+++ b/ThingConnect.Pulse.Server/Services/Monitoring/OutageDetectionService.cs
@@ -14,12 +14,14 @@ public sealed class OutageDetectionService : IOutageDetectionService
{
private readonly IServiceProvider _serviceProvider;
private readonly ILogger _logger;
+ private readonly ICheckResultWriteQueue _writeQueue;
private readonly ConcurrentDictionary _states = new();
- public OutageDetectionService(IServiceProvider serviceProvider, ILogger logger)
+ public OutageDetectionService(IServiceProvider serviceProvider, ILogger logger, ICheckResultWriteQueue writeQueue)
{
_serviceProvider = serviceProvider;
_logger = logger;
+ _writeQueue = writeQueue;
}
public async Task ProcessCheckResultAsync(CheckResult result, CancellationToken cancellationToken = default)
@@ -69,8 +71,8 @@ public async Task ProcessCheckResultAsync(CheckResult result, Cancellation
result.EndpointId, state.SuccessStreak);
}
- // Save the raw check result to database (independent of transitions)
- await SaveCheckResultAsync(result, cancellationToken);
+ // Enqueue raw check result — written by the background write queue (non-blocking)
+ SaveCheckResult(result);
return stateChanged;
}
@@ -515,11 +517,8 @@ private async Task UpdateEndpointStatusAsync(PulseDbContext context, Guid endpoi
return (endpointStatus, openOutageId, false);
}
- private async Task SaveCheckResultAsync(CheckResult result, CancellationToken cancellationToken)
+ private void SaveCheckResult(CheckResult result)
{
- using IServiceScope scope = _serviceProvider.CreateScope();
- PulseDbContext context = scope.ServiceProvider.GetRequiredService();
-
CheckResultRaw rawResult = new CheckResultRaw
{
EndpointId = result.EndpointId,
@@ -529,21 +528,10 @@ private async Task SaveCheckResultAsync(CheckResult result, CancellationToken ca
Error = result.Error
};
- context.CheckResultsRaw.Add(rawResult);
+ Guid? endpointIdForRtt = result.Status == UpDown.up && result.RttMs.HasValue
+ ? result.EndpointId
+ : null;
- // Update LastRttMs for successful probes
- if (result.Status == UpDown.up && result.RttMs.HasValue)
- {
- Data.Endpoint? endpoint = await context.Endpoints.FindAsync([result.EndpointId], cancellationToken);
- if (endpoint != null)
- {
- endpoint.LastRttMs = result.RttMs.Value;
- _logger.LogInformation(
- "Updated LastRttMs for endpoint {EndpointId} to {RttMs}ms",
- result.EndpointId, result.RttMs.Value);
- }
- }
-
- await context.SaveChangesAsync(cancellationToken);
+ _writeQueue.Enqueue(new WriteQueueItem(rawResult, endpointIdForRtt, result.RttMs));
}
}
diff --git a/ThingConnect.Pulse.Server/Services/Prune/PruneBackgroundService.cs b/ThingConnect.Pulse.Server/Services/Prune/PruneBackgroundService.cs
new file mode 100644
index 0000000..07d83c7
--- /dev/null
+++ b/ThingConnect.Pulse.Server/Services/Prune/PruneBackgroundService.cs
@@ -0,0 +1,90 @@
+namespace ThingConnect.Pulse.Server.Services.Prune;
+
+public sealed class PruneBackgroundService : BackgroundService
+{
+ private readonly IServiceProvider _serviceProvider;
+ private readonly IConfiguration _configuration;
+ private readonly ILogger _logger;
+
+ public PruneBackgroundService(
+ IServiceProvider serviceProvider,
+ IConfiguration configuration,
+ ILogger logger)
+ {
+ _serviceProvider = serviceProvider;
+ _configuration = configuration;
+ _logger = logger;
+ }
+
+ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+ {
+ // Wait until the first 02:00 local time before running, then repeat every intervalHours
+ int intervalHours = _configuration.GetValue("Data:Pruning:IntervalHours", 24);
+ bool enabled = _configuration.GetValue("Data:Pruning:Enabled", true);
+
+ if (!enabled)
+ {
+ _logger.LogInformation("Pruning is disabled via configuration");
+ return;
+ }
+
+ // Initial delay: time until next 02:00 local
+ TimeSpan initialDelay = TimeUntilNextRun(TimeSpan.FromHours(2));
+ _logger.LogInformation(
+ "Prune background service scheduled. First run in {Delay:hh\\:mm\\:ss}, then every {IntervalHours}h",
+ initialDelay, intervalHours);
+
+ try
+ {
+ await Task.Delay(initialDelay, stoppingToken);
+ }
+ catch (OperationCanceledException)
+ {
+ return;
+ }
+
+ while (!stoppingToken.IsCancellationRequested)
+ {
+ await RunPruneAsync(stoppingToken);
+
+ try
+ {
+ await Task.Delay(TimeSpan.FromHours(intervalHours), stoppingToken);
+ }
+ catch (OperationCanceledException)
+ {
+ break;
+ }
+ }
+ }
+
+ private async Task RunPruneAsync(CancellationToken ct)
+ {
+ _logger.LogInformation("Starting scheduled raw data prune");
+ var sw = System.Diagnostics.Stopwatch.StartNew();
+
+ try
+ {
+ using IServiceScope scope = _serviceProvider.CreateScope();
+ IPruneService pruneService = scope.ServiceProvider.GetRequiredService();
+
+ int deleted = await pruneService.PruneRawDataAsync(dryRun: false, ct);
+
+ _logger.LogInformation(
+ "Scheduled prune complete: deleted {Deleted} raw check results in {Elapsed}ms",
+ deleted, sw.ElapsedMilliseconds);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError(ex, "Scheduled prune failed after {Elapsed}ms", sw.ElapsedMilliseconds);
+ }
+ }
+
+ private static TimeSpan TimeUntilNextRun(TimeSpan targetTimeOfDay)
+ {
+ DateTimeOffset now = DateTimeOffset.Now;
+ DateTimeOffset next = now.Date + targetTimeOfDay;
+ if (next <= now) next = next.AddDays(1);
+ return next - now;
+ }
+}
diff --git a/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs b/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs
index c49e9b1..b66fa70 100644
--- a/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs
+++ b/ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs
@@ -33,14 +33,13 @@ public async Task ProcessRollup15mAsync(CancellationToken cancellationToken = de
_logger.LogDebug("Processing 15m rollups from {FromTs} to {ToTs}", UnixTimestamp.FromUnixSeconds(fromTs), UnixTimestamp.FromUnixSeconds(toTs));
- // Get all raw checks in the time window
- // SQLite has issues with DateTimeOffset comparisons in LINQ, so fetch all and filter in memory
- List allChecks = await _context.CheckResultsRaw.ToListAsync(cancellationToken);
- var rawChecks = allChecks
+ // Get raw checks in the time window — Ts is a long (unix seconds), safe to filter in SQL
+ List rawChecks = await _context.CheckResultsRaw
.Where(c => c.Ts > fromTs && c.Ts <= toTs)
+ .AsNoTracking()
.OrderBy(c => c.EndpointId)
.ThenBy(c => c.Ts)
- .ToList();
+ .ToListAsync(cancellationToken);
if (!rawChecks.Any())
{
@@ -95,17 +94,16 @@ public async Task ProcessRollupDailyAsync(CancellationToken cancellationToken =
_logger.LogDebug("Processing daily rollups from {FromDate} to {ToDate}", fromDate, toDate);
- // Get all raw checks in the date range
+ // Get raw checks in the date range — Ts is a long (unix seconds), safe to filter in SQL
long fromTs = UnixTimestamp.ToUnixDate(fromDate);
long toTs = UnixTimestamp.ToUnixDate(toDate);
- // SQLite has issues with DateTimeOffset comparisons in LINQ, so fetch all and filter in memory
- List allChecks = await _context.CheckResultsRaw.ToListAsync(cancellationToken);
- var rawChecks = allChecks
+ List rawChecks = await _context.CheckResultsRaw
.Where(c => c.Ts >= fromTs && c.Ts < toTs)
+ .AsNoTracking()
.OrderBy(c => c.EndpointId)
.ThenBy(c => c.Ts)
- .ToList();
+ .ToListAsync(cancellationToken);
if (!rawChecks.Any())
{
diff --git a/ThingConnect.Pulse.Server/Services/StatusService.cs b/ThingConnect.Pulse.Server/Services/StatusService.cs
index 3fade8c..9e64cf6 100644
--- a/ThingConnect.Pulse.Server/Services/StatusService.cs
+++ b/ThingConnect.Pulse.Server/Services/StatusService.cs
@@ -50,46 +50,35 @@ public async Task> GetLiveStatusAsync(string? group, str
e.Host.ToLower().Contains(searchLower));
}
- // Get total count for pagination
- int totalCount = await query.CountAsync();
-
- // Apply pagination
List endpoints = await query
- .OrderBy(e => e.GroupId)
- .ThenBy(e => e.Name)
- .ToListAsync();
+ .OrderBy(e => e.GroupId)
+ .ThenBy(e => e.Name)
+ .ToListAsync();
- // Get live status for each endpoint
var items = new List();
var endpointIds = endpoints.Select(e => e.Id).ToList();
- // Get latest checks for all endpoints - optimized query using window functions in SQLite
- var latestChecks = await _context.CheckResultsRaw
- .Where(c => endpointIds.Contains(c.EndpointId))
- .AsNoTracking()
- .GroupBy(c => c.EndpointId)
- .Select(g => new
- {
- EndpointId = g.Key,
- LatestCheck = g.OrderByDescending(c => c.Ts).FirstOrDefault()
- })
- .ToListAsync();
-
- var latestCheckDict = latestChecks.ToDictionary(x => x.EndpointId, x => x.LatestCheck);
-
- // Get sparkline data (last 20 checks per endpoint for mini chart)
+ // Sparkline query (2h window) — last point also serves as latest check timestamp
Dictionary> sparklineData = await GetSparklineDataAsync(endpointIds);
+ // Single 5min query for flap detection
+ HashSet flappingIds = await GetFlappingEndpointIdsAsync(endpointIds);
+
foreach (Data.Endpoint? endpoint in endpoints)
{
- StatusType status = DetermineStatus(endpoint, latestCheckDict);
- List sparkline = sparklineData.ContainsKey(endpoint.Id)
- ? sparklineData[endpoint.Id]
- : new List();
+ List sparkline = sparklineData.GetValueOrDefault(endpoint.Id, new List());
+
+ // Most recent sparkline point gives us the last check time — no extra check_result_raw query needed
+ long? latestCheckTs = sparkline.Count > 0
+ ? UnixTimestamp.ToUnixSeconds(sparkline.Last().Ts)
+ : null;
- _logger.LogInformation(
- "Endpoint {EndpointName}: Status = {Status}, LastRttMs = {RttMs}, LastChangeTs = {LastChangeTs}",
- endpoint.Name, status, endpoint.LastRttMs, endpoint.LastChangeTs);
+ // endpoint.LastStatus/LastRttMs/LastChangeTs are maintained by OutageDetectionService
+ StatusType status = DetermineStatus(endpoint, latestCheckTs, flappingIds);
+
+ _logger.LogDebug(
+ "Endpoint {EndpointName}: Status={Status}, RttMs={RttMs}",
+ endpoint.Name, status, endpoint.LastRttMs);
items.Add(new LiveStatusItemDto
{
@@ -180,61 +169,54 @@ private async Task>> GetSparklineDataAsync
return sparklineData;
}
- private StatusType DetermineStatus(Data.Endpoint endpoint, Dictionary latestChecks)
+ private StatusType DetermineStatus(Data.Endpoint endpoint, long? latestCheckTs, HashSet flappingIds)
{
- // Check if we have recent check data
- if (!latestChecks.TryGetValue(endpoint.Id, out CheckResultRaw? latestCheck) || latestCheck == null)
+ if (latestCheckTs == null)
{
- return StatusType.Down; // No data means down
+ return StatusType.Down;
}
- // Check if the latest check is recent enough (within 2x interval)
- var expectedInterval = TimeSpan.FromSeconds(endpoint.IntervalSeconds * 2);
- if (UnixTimestamp.Now() - latestCheck.Ts > (long)expectedInterval.TotalSeconds)
+ if (UnixTimestamp.Now() - latestCheckTs.Value > endpoint.IntervalSeconds * 2)
{
- return StatusType.Down; // Stale data means down
+ return StatusType.Down;
}
- // Check for flapping (multiple state changes in short period)
- // This is simplified - in production you'd want more sophisticated flap detection
- if (IsFlapping(endpoint.Id).Result)
+ if (flappingIds.Contains(endpoint.Id))
{
return StatusType.Flapping;
}
- return latestCheck.Status == UpDown.up ? StatusType.Up : StatusType.Down;
+ return endpoint.LastStatus == UpDown.up ? StatusType.Up : StatusType.Down;
}
- private async Task IsFlapping(Guid endpointId)
+ // Single query for all endpoints instead of one query per endpoint with .Result (which causes thread pool starvation)
+ private async Task> GetFlappingEndpointIdsAsync(List endpointIds)
{
- // Simple flap detection: check if there were > 3 state changes in last 5 minutes
long cutoffTime = UnixTimestamp.Subtract(UnixTimestamp.Now(), TimeSpan.FromMinutes(5));
- var checks = await _context.CheckResultsRaw
- .Where(c => c.EndpointId == endpointId && c.Ts >= cutoffTime)
+
+ var recentChecks = await _context.CheckResultsRaw
+ .Where(c => endpointIds.Contains(c.EndpointId) && c.Ts >= cutoffTime)
.AsNoTracking()
- .Select(c => new { c.Ts, c.Status })
+ .Select(c => new { c.EndpointId, c.Ts, c.Status })
.ToListAsync();
- var recentChecks = checks
- .OrderBy(c => c.Ts)
- .Select(c => c.Status)
- .ToList();
+ var flapping = new HashSet();
- if (recentChecks.Count < 4)
+ foreach (var group in recentChecks.GroupBy(c => c.EndpointId))
{
- return false;
- }
+ var statuses = group.OrderBy(c => c.Ts).Select(c => c.Status).ToList();
+ if (statuses.Count < 4) continue;
- int stateChanges = 0;
- for (int i = 1; i < recentChecks.Count; i++)
- {
- if (recentChecks[i] != recentChecks[i - 1])
+ int changes = 0;
+ for (int i = 1; i < statuses.Count; i++)
{
- stateChanges++;
+ if (statuses[i] != statuses[i - 1]) changes++;
}
+
+ if (changes > 3) flapping.Add(group.Key);
}
- return stateChanges > 3;
+ return flapping;
}
private EndpointDto MapToEndpointDto(Data.Endpoint endpoint)
diff --git a/ThingConnect.Pulse.Server/appsettings.json b/ThingConnect.Pulse.Server/appsettings.json
index 1913b32..69dc01c 100644
--- a/ThingConnect.Pulse.Server/appsettings.json
+++ b/ThingConnect.Pulse.Server/appsettings.json
@@ -1,13 +1,15 @@
{
"Serilog": {
- "Using": ["Serilog.Sinks.Console", "Serilog.Sinks.File"],
+ "Using": [
+ "Serilog.Sinks.Console",
+ "Serilog.Sinks.File"
+ ],
"MinimumLevel": {
"Default": "Information",
"Override": {
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information",
"Microsoft.EntityFrameworkCore": "Warning",
- "Microsoft.EntityFrameworkCore.Database.Command": "Debug",
"System": "Warning"
}
},
@@ -31,7 +33,12 @@
}
}
],
- "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"],
+ "Enrich": [
+ "FromLogContext",
+ "WithMachineName",
+ "WithProcessId",
+ "WithThreadId"
+ ],
"Properties": {
"Application": "ThingConnect.Pulse",
"Environment": "Development"
@@ -80,4 +87,4 @@
"Monitoring": {
"MaxConcurrentProbes": 100
}
-}
+}
\ No newline at end of file
diff --git a/ThingConnect.Pulse.Tests/CidrExpansionTests.cs b/ThingConnect.Pulse.Tests/CidrExpansionTests.cs
index d16c77a..abe18f8 100644
--- a/ThingConnect.Pulse.Tests/CidrExpansionTests.cs
+++ b/ThingConnect.Pulse.Tests/CidrExpansionTests.cs
@@ -36,7 +36,8 @@ private IEnumerable ExpandCidr(string cidr)
}
byte[] addressBytes = ipAddress.GetAddressBytes();
- uint addressInt = BitConverter.ToUInt32(addressBytes.Reverse().ToArray(), 0);
+ Array.Reverse(addressBytes);
+ uint addressInt = BitConverter.ToUInt32(addressBytes, 0);
int hostBits = 32 - prefixLength;
uint hostCount = (uint)(1 << hostBits);
@@ -48,7 +49,8 @@ private IEnumerable ExpandCidr(string cidr)
for (uint address = startAddress; address < endAddress && address > networkAddress; address++)
{
- byte[] bytes = BitConverter.GetBytes(address).Reverse().ToArray();
+ byte[] bytes = BitConverter.GetBytes(address);
+ Array.Reverse(bytes);
var ip = new IPAddress(bytes);
yield return ip.ToString();
}
diff --git a/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs b/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs
index 5941efd..05b6748 100644
--- a/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs
+++ b/ThingConnect.Pulse.Tests/ConfigurationValidationTests.cs
@@ -151,7 +151,8 @@ private List ExpandCidrForTesting(string cidr)
}
byte[] addressBytes = ipAddress.GetAddressBytes();
- uint addressInt = BitConverter.ToUInt32(addressBytes.Reverse().ToArray(), 0);
+ Array.Reverse(addressBytes);
+ uint addressInt = BitConverter.ToUInt32(addressBytes, 0);
int hostBits = 32 - prefixLength;
uint hostCount = (uint)(1 << hostBits);
@@ -163,7 +164,8 @@ private List ExpandCidrForTesting(string cidr)
for (uint address = startAddress; address < endAddress && address > networkAddress; address++)
{
- byte[] bytes = BitConverter.GetBytes(address).Reverse().ToArray();
+ byte[] bytes = BitConverter.GetBytes(address);
+ Array.Reverse(bytes);
var ip = new System.Net.IPAddress(bytes);
result.Add(ip.ToString());
}
diff --git a/docs/fix-ram-usage.md b/docs/fix-ram-usage.md
new file mode 100644
index 0000000..e15e5b3
--- /dev/null
+++ b/docs/fix-ram-usage.md
@@ -0,0 +1,91 @@
+# Fix: High RAM Usage (~70% on Windows Service)
+
+## Root Causes
+
+### 1. Thundering herd in MonitoringBackgroundService ✅ Fixed
+**File:** `ThingConnect.Pulse.Server/Services/Monitoring/MonitoringBackgroundService.cs`
+
+Every 15 seconds the refresh loop restarted **all** endpoint timers with `dueTime: TimeSpan.Zero`, firing every probe simultaneously regardless of configured intervals. With 200 concurrent probes, this caused bursts of 200 EF Core DbContext instances + tasks every 15 seconds.
+
+**Fix applied:** Timers are now only restarted when the interval changes. Existing timers run undisturbed on their own schedule.
+
+---
+
+### 2. MaxConcurrentProbes too high ✅ Fixed
+**File:** `ThingConnect.Pulse.Server/appsettings.json`
+
+`MaxConcurrentProbes` was set to 200. Each concurrent probe creates a DbContext (~50–100KB each) plus async task overhead.
+
+**Fix applied:** Lowered to 100.
+
+---
+
+### 3. EF Core SQL debug logging ✅ Fixed
+**File:** `ThingConnect.Pulse.Server/appsettings.json`
+
+`"Microsoft.EntityFrameworkCore.Database.Command": "Debug"` was logging every SQL query. With hundreds of probes per minute, Serilog's internal buffer accumulated thousands of log strings per minute.
+
+**Fix applied:** Removed the debug override, leaving EF Core at `Warning`.
+
+---
+
+### 4. RollupService loads entire check_results_raw table every 5 minutes ❌ Not fixed yet
+**File:** `ThingConnect.Pulse.Server/Services/Rollup/RollupService.cs` — Lines 38, 103
+
+```csharp
+// PROBLEM: loads every row in the table into memory
+List allChecks = await _context.CheckResultsRaw.ToListAsync(cancellationToken);
+var rawChecks = allChecks.Where(c => c.Ts > fromTs && c.Ts <= toTs)...
+```
+
+The comment says "SQLite has issues with DateTimeOffset comparisons" but `Ts` is stored as a `long` unix timestamp — EF Core pushes `long` comparisons to SQL without issue. With 60-day retention at moderate probe frequency this is potentially millions of rows loaded every 5 minutes.
+
+**Fix needed:**
+```csharp
+List rawChecks = await _context.CheckResultsRaw
+ .Where(c => c.Ts > fromTs && c.Ts <= toTs)
+ .AsNoTracking()
+ .OrderBy(c => c.EndpointId).ThenBy(c => c.Ts)
+ .ToListAsync(cancellationToken);
+```
+Apply the same change to both `ProcessRollup15mAsync` (line 38) and `ProcessRollupDailyAsync` (line 103).
+
+---
+
+### 5. HistoryService loads all rows for an endpoint then filters in memory ❌ Not fixed yet
+**File:** `ThingConnect.Pulse.Server/Services/HistoryService.cs` — Lines 88, 112, 137, 161
+
+All four query methods (`GetRawDataAsync`, `GetRollup15mDataAsync`, `GetRollupDailyDataAsync`, `GetOutagesAsync`) follow the same pattern:
+
+```csharp
+// PROBLEM: loads all rows for the endpoint, filters by time in C#
+var data = await _context.CheckResultsRaw
+ .Where(c => c.EndpointId == endpointId)
+ .ToListAsync();
+return data.Where(c => c.Ts >= fromUnix && c.Ts <= toUnix)...
+```
+
+**Fix needed:** Push the time filter into SQL and add `AsNoTracking()`:
+```csharp
+return await _context.CheckResultsRaw
+ .Where(c => c.EndpointId == endpointId && c.Ts >= fromUnix && c.Ts <= toUnix)
+ .AsNoTracking()
+ .OrderBy(c => c.Ts)
+ .Select(c => new RawCheckDto { ... })
+ .ToListAsync();
+```
+Apply to all four methods — same pattern, just add the timestamp condition into the existing `.Where()`.
+
+---
+
+## Priority
+
+| # | Issue | Impact | Status |
+|---|-------|--------|--------|
+| 1 | Thundering herd (timer restart) | High | ✅ Fixed |
+| 2 | MaxConcurrentProbes = 200 | Medium | ✅ Fixed |
+| 3 | EF Core debug logging | Medium | ✅ Fixed |
+| 4 | RollupService full table scan | **Critical** | ✅ Fixed |
+| 5 | HistoryService full scan per request | High | ✅ Fixed |
+
+All five issues resolved.
diff --git a/setup.iss b/setup.iss
index 24ac026..7ac814a 100644
--- a/setup.iss
+++ b/setup.iss
@@ -23,7 +23,7 @@ DefaultDirName={autopf}\ThingConnect.Pulse
DefaultGroupName={#AppName}
DisableProgramGroupPage=yes
OutputDir=installer
-OutputBaseFilename=ThingConnect Pulse - Setup {#AppVersion}
+OutputBaseFilename=ThingConnect.Pulse.Setup {#AppVersion}
Compression=lzma
SolidCompression=yes
WizardStyle=modern
diff --git a/thingconnect.pulse.client/vite.config.ts b/thingconnect.pulse.client/vite.config.ts
index 62f79ec..452b44f 100644
--- a/thingconnect.pulse.client/vite.config.ts
+++ b/thingconnect.pulse.client/vite.config.ts
@@ -38,7 +38,7 @@ const target = env.ASPNETCORE_HTTPS_PORT
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
: env.ASPNETCORE_URLS
? env.ASPNETCORE_URLS.split(';')[0]
- : 'http://localhost:8080';
+ : 'http://localhost:8090';
// https://vitejs.dev/config/
export default defineConfig({
@@ -49,10 +49,12 @@ export default defineConfig({
},
},
server: {
+ host: '0.0.0.0',
proxy: {
'^/api': {
target,
secure: false,
+ changeOrigin: true,
},
},
port: parseInt(env.DEV_SERVER_PORT || '55605'),