VPR-54 feat(scheduler): Hangfire-backed scheduler with RAPS role-refresh

Adds a shared, area-agnostic background-job scheduler on top of
Hangfire 1.8 + SQL Server, with a CAS+RAPS-gated dashboard and a thin
[ScheduledJob] attribute for job authors. The RAPS nightly role-refresh
is the first consumer; the legacy ColdFusion runner is now a
debug-only manual fallback.

- Hangfire storage lives in the VIPER DB under the [HangFire] schema;
  Hangfire:Enabled (default true) is the master switch and
  Hangfire:AutoSchedule (dev: false) registers jobs with Cron.Never so
  developers don't run cron locally.
- Dashboard at /scheduler/dashboard is gated by SVMSecure.CATS.scheduledJobs
  (same RAPS permission as the legacy cats/inc_scheduledTasks.cfm
  scheduler, so existing admins inherit access). Hangfire.Console and
  Hangfire.Heartbeat plugins surface per-job logs and CPU/RAM metrics.
- Jobs implement IScheduledJob and carry [ScheduledJob(id, cron)]. A
  startup discovery pass scans the web assembly, registers each type
  with DI, and AddOrUpdate's the recurring job through a single
  ScheduledJobRunner dispatcher (Hangfire can't serialize calls against
  interfaces). Each execution gets a fresh DI scope and a
  ScheduledJobContext exposing TriggerSource + ModBy ("__sched" for
  scheduled runs, LoginId for manual).
- Hangfire health check tagged "ready" piggybacks on /health/detail:
  reports Healthy / Degraded / Unhealthy based on storage reachability
  and recent server heartbeats.
- RAPS role-refresh (raps:role-refresh, 0 0 * * * Pacific) ports the
  legacy ColdFusion runner. RoleViews.UpdateRoles now accepts an
  explicit modBy so scheduled and manual paths stamp audit rows
  distinguishably without UserHelper.GetCurrentUser().
- Program.cs Cloudflare/F5 forwarded-headers block is extracted to
  ForwardedHeadersExtensions to keep Main$ below CA1502/CA1505 limits
  with the new AddViperHangfire wiring added alongside it.

Author: rlorenzo

.env.local.example

@@ -15,6 +15,13 @@
 # Default: C:\Tools\mailpit
 #MAILPIT_INSTALL_DIR=C:\Tools\mailpit
 
+# Hangfire Scheduler (Optional, Off by Default)
+# Master toggle for the background scheduler. When true, the app registers
+# Hangfire and starts the background server; when false (default) Hangfire
+# is a no-op. Hangfire's tables live in the VIPER database under the
+# HangFire schema.
+#Hangfire__Enabled=false
+
 # Jenkins Build Trigger (pre-push hook)
 # Get your API token from Jenkins: User menu > Configure > API Token
 #JENKINS_USER=your-username

package-lock.json

@@ -0,0 +1,166 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using NSubstitute;
+using Viper.Classes.HealthChecks;
+
+namespace Viper.test.HealthChecks
+{
+    public class HangfireHealthCheckTests
+    {
+        private static HealthCheckContext CreateContext(HangfireHealthCheck sut)
+        {
+            return new HealthCheckContext
+            {
+                Registration = new HealthCheckRegistration("hangfire", sut, null, null)
+            };
+        }
+
+        private static (Hangfire.JobStorage storage, Hangfire.Storage.IMonitoringApi monitoring) CreateStorage()
+        {
+            var monitoring = Substitute.For<Hangfire.Storage.IMonitoringApi>();
+            var storage = Substitute.For<Hangfire.JobStorage>();
+            storage.GetMonitoringApi().Returns(monitoring);
+            return (storage, monitoring);
+        }
+
+        private static Hangfire.Storage.Monitoring.StatisticsDto SampleStats(long servers = 1) => new()
+        {
+            Servers = servers,
+            Enqueued = 2,
+            Scheduled = 3,
+            Processing = 4,
+            Failed = 5,
+            Recurring = 6
+        };
+
+        [Fact]
+        public async Task CheckHealthAsync_HealthyWhenServersHaveRecentHeartbeats()
+        {
+            var (storage, monitoring) = CreateStorage();
+            monitoring.GetStatistics().Returns(SampleStats());
+            monitoring.Servers().Returns(new List<Hangfire.Storage.Monitoring.ServerDto>
+            {
+                new() { Name = "srv-1", Heartbeat = DateTime.UtcNow }
+            });
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Equal(HealthStatus.Healthy, result.Status);
+            Assert.Contains("Hangfire OK", result.Description);
+            Assert.Equal(1L, result.Data["servers"]);
+        }
+
+        [Fact]
+        public async Task CheckHealthAsync_DegradedWhenNoServersRegistered()
+        {
+            var (storage, monitoring) = CreateStorage();
+            monitoring.GetStatistics().Returns(SampleStats(0));
+            monitoring.Servers().Returns(new List<Hangfire.Storage.Monitoring.ServerDto>());
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Equal(HealthStatus.Degraded, result.Status);
+            Assert.Contains("no servers registered", result.Description);
+        }
+
+        [Fact]
+        public async Task CheckHealthAsync_UnhealthyWhenAllHeartbeatsStale()
+        {
+            var (storage, monitoring) = CreateStorage();
+            monitoring.GetStatistics().Returns(SampleStats());
+            monitoring.Servers().Returns(new List<Hangfire.Storage.Monitoring.ServerDto>
+            {
+                new() { Name = "srv-stale", Heartbeat = DateTime.UtcNow.AddMinutes(-10) }
+            });
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Equal(HealthStatus.Unhealthy, result.Status);
+            Assert.Contains("stale", result.Description);
+        }
+
+        [Fact]
+        public async Task CheckHealthAsync_UnhealthyWhenServerHasNullHeartbeat()
+        {
+            var (storage, monitoring) = CreateStorage();
+            monitoring.GetStatistics().Returns(SampleStats());
+            monitoring.Servers().Returns(new List<Hangfire.Storage.Monitoring.ServerDto>
+            {
+                new() { Name = "srv-never", Heartbeat = null }
+            });
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Equal(HealthStatus.Unhealthy, result.Status);
+            Assert.Contains("never", result.Description);
+        }
+
+        [Fact]
+        public async Task CheckHealthAsync_HealthyWhenAtLeastOneHeartbeatIsRecent()
+        {
+            var (storage, monitoring) = CreateStorage();
+            monitoring.GetStatistics().Returns(SampleStats(2));
+            monitoring.Servers().Returns(new List<Hangfire.Storage.Monitoring.ServerDto>
+            {
+                new() { Name = "srv-stale", Heartbeat = DateTime.UtcNow.AddMinutes(-10) },
+                new() { Name = "srv-fresh", Heartbeat = DateTime.UtcNow }
+            });
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Equal(HealthStatus.Healthy, result.Status);
+            Assert.Contains("Hangfire OK", result.Description);
+        }
+
+        [Fact]
+        public async Task CheckHealthAsync_UnhealthyWhenStorageThrows()
+        {
+            var storage = Substitute.For<Hangfire.JobStorage>();
+            var boom = new InvalidOperationException("boom");
+            storage.GetMonitoringApi().Returns(_ => throw boom);
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Equal(HealthStatus.Unhealthy, result.Status);
+            Assert.Contains("unreachable", result.Description);
+            Assert.Same(boom, result.Exception);
+        }
+
+        [Fact]
+        public async Task CheckHealthAsync_DataDictionaryContainsAllStatsKeys()
+        {
+            var (storage, monitoring) = CreateStorage();
+            monitoring.GetStatistics().Returns(new Hangfire.Storage.Monitoring.StatisticsDto
+            {
+                Servers = 1,
+                Enqueued = 7,
+                Scheduled = 8,
+                Processing = 9,
+                Failed = 10,
+                Recurring = 11
+            });
+            monitoring.Servers().Returns(new List<Hangfire.Storage.Monitoring.ServerDto>
+            {
+                new() { Name = "srv-1", Heartbeat = DateTime.UtcNow }
+            });
+
+            var sut = new HangfireHealthCheck(storage);
+            var result = await sut.CheckHealthAsync(CreateContext(sut));
+
+            Assert.Contains("servers", result.Data.Keys);
+            Assert.Contains("enqueued", result.Data.Keys);
+            Assert.Contains("scheduled", result.Data.Keys);
+            Assert.Contains("processing", result.Data.Keys);
+            Assert.Contains("failed", result.Data.Keys);
+            Assert.Contains("recurring", result.Data.Keys);
+            Assert.Equal(1L, result.Data["servers"]);
+            Assert.Equal(7L, result.Data["enqueued"]);
+            Assert.Equal(11L, result.Data["recurring"]);
+        }
+    }
+}

test/HealthChecks/HangfireHealthCheckTests.cs

@@ -0,0 +1,20 @@
+using System.Reflection;
+using Viper.Areas.RAPS.Jobs;
+using Viper.Areas.Scheduler.Services;
+
+namespace Viper.test.RAPS
+{
+    public sealed class RapsRoleRefreshScheduledJobTests
+    {
+        [Fact]
+        public void Class_IsDecoratedWithScheduledJob()
+        {
+            var attr = typeof(RapsRoleRefreshScheduledJob).GetCustomAttribute<ScheduledJobAttribute>();
+
+            Assert.NotNull(attr);
+            Assert.Equal("raps:role-refresh", attr.Id);
+            Assert.Equal("0 0 * * *", attr.Cron);
+            Assert.Equal("Pacific Standard Time", attr.TimeZoneId);
+        }
+    }
+}