VPR-141 feat(healthchecks): add /health endpoints and UI dashboard

- /health: anonymous liveness for Jenkins probes.
- /health/detail: per-check JSON (UI format) gated by InternalAllowlist
  (developer IPs only). Intentionally not CAS-gated so it stays
  reachable when auth subsystems are degraded.
- /healthchecks: HealthChecks.UI dashboard, IP-gated to the same list.
- Database, disk, AWS SSM, LDAP, CAS, SMTP, and VMACs probes; campus-*
  external probes wrapped in adaptive polling so a healthy check is
  cached for an hour and a failing one re-probes every 5 min.
- The in-app HealthChecksUI collector polls /health/detail through
  the IP filter without widening the allowlist: a process-startup
  random token is stamped on every outbound poll by a delegating
  handler (UseApiEndpointDelegatingHandler), and the endpoint filter
  bypasses the IP check on a constant-time match. Token cycles per
  process, so it can't be replayed across deploys.
- Unit tests cover the constant-time matcher, header lookup
  (case-insensitive), and the delegating handler's stamp/replace/
  forward behaviors.

Author: rlorenzo

JenkinsFile

@@ -109,6 +109,27 @@ pipeline {
 				networkShares()
 				filecopy copyOfflineOperations, 'test', env.WORKSPACE
 				filecopy copyOperations, 'test', env.WORKSPACE
+				powershell label: 'Verify /health returns 200', script: '''
+					$url = "https://secure-test.vetmed.ucdavis.edu/2/health"
+					$maxAttempts = 15
+					for ($i = 1; $i -le $maxAttempts; $i++) {
+						# 2s for the first few attempts then 4s - absorbs IIS app-pool warm-up
+						$delay = if ($i -le 5) { 2 } else { 4 }
+						try {
+							$response = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 10
+							if ($response.StatusCode -eq 200) {
+								Write-Host "Attempt ${i}: $url returned 200 OK"
+								exit 0
+							}
+							Write-Host "Attempt ${i}: $url returned $($response.StatusCode)"
+						} catch {
+							Write-Host "Attempt ${i}: $($_.Exception.Message)"
+						}
+						if ($i -lt $maxAttempts) { Start-Sleep -Seconds $delay }
+					}
+					Write-Error "Health check at $url failed after $maxAttempts attempts"
+					exit 1
+				'''
 			}
 		}
 		stage('Deploy to prod') {
@@ -122,6 +143,26 @@ pipeline {
 				networkShares()
 				filecopy copyOfflineOperations, 'prod', env.WORKSPACE
 				filecopy copyOperations, 'prod', env.WORKSPACE
+				powershell label: 'Verify /health returns 200', script: '''
+					$url = "https://secure.vetmed.ucdavis.edu/2/health"
+					$maxAttempts = 15
+					for ($i = 1; $i -le $maxAttempts; $i++) {
+						$delay = if ($i -le 5) { 2 } else { 4 }
+						try {
+							$response = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 10
+							if ($response.StatusCode -eq 200) {
+								Write-Host "Attempt ${i}: $url returned 200 OK"
+								exit 0
+							}
+							Write-Host "Attempt ${i}: $url returned $($response.StatusCode)"
+						} catch {
+							Write-Host "Attempt ${i}: $($_.Exception.Message)"
+						}
+						if ($i -lt $maxAttempts) { Start-Sleep -Seconds $delay }
+					}
+					Write-Error "Health check at $url failed after $maxAttempts attempts"
+					exit 1
+				'''
 			}
 		}
     }

test/HealthChecks/HealthCheckCollectorAuthTests.cs

@@ -0,0 +1,81 @@
+using Microsoft.AspNetCore.Http;
+using Viper.Classes.HealthChecks;
+
+namespace Viper.test.HealthChecks
+{
+    public class HealthCheckCollectorAuthTests
+    {
+        [Fact]
+        public void Matches_NullProvided_ReturnsFalse()
+        {
+            Assert.False(HealthCheckCollectorAuth.Matches(null));
+        }
+
+        [Fact]
+        public void Matches_EmptyProvided_ReturnsFalse()
+        {
+            Assert.False(HealthCheckCollectorAuth.Matches(string.Empty));
+        }
+
+        [Fact]
+        public void Matches_DifferentLength_ReturnsFalse()
+        {
+            Assert.False(HealthCheckCollectorAuth.Matches("short"));
+        }
+
+        [Fact]
+        public void Matches_SameLengthDifferentValue_ReturnsFalse()
+        {
+            // Same length as the real token (32 hex chars from Guid.ToString("N")) but different bytes.
+            var sameLengthFake = new string('a', HealthCheckCollectorAuth.Token.Length);
+            Assert.False(HealthCheckCollectorAuth.Matches(sameLengthFake));
+        }
+
+        [Fact]
+        public void Matches_RealToken_ReturnsTrue()
+        {
+            Assert.True(HealthCheckCollectorAuth.Matches(HealthCheckCollectorAuth.Token));
+        }
+
+        [Fact]
+        public void IsCollectorRequest_HeaderAbsent_ReturnsFalse()
+        {
+            var ctx = new DefaultHttpContext();
+            Assert.False(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
+        }
+
+        [Fact]
+        public void IsCollectorRequest_HeaderEmpty_ReturnsFalse()
+        {
+            var ctx = new DefaultHttpContext();
+            ctx.Request.Headers[HealthCheckCollectorAuth.HeaderName] = string.Empty;
+            Assert.False(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
+        }
+
+        [Fact]
+        public void IsCollectorRequest_WrongToken_ReturnsFalse()
+        {
+            var ctx = new DefaultHttpContext();
+            ctx.Request.Headers[HealthCheckCollectorAuth.HeaderName] = "not-the-real-token";
+            Assert.False(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
+        }
+
+        [Fact]
+        public void IsCollectorRequest_ValidToken_ReturnsTrue()
+        {
+            var ctx = new DefaultHttpContext();
+            ctx.Request.Headers[HealthCheckCollectorAuth.HeaderName] = HealthCheckCollectorAuth.Token;
+            Assert.True(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
+        }
+
+        [Fact]
+        public void IsCollectorRequest_HeaderNameIsCaseInsensitive()
+        {
+            // ASP.NET request headers are case-insensitive; the filter must accept any casing
+            // a proxy might apply.
+            var ctx = new DefaultHttpContext();
+            ctx.Request.Headers["x-health-collector-token"] = HealthCheckCollectorAuth.Token;
+            Assert.True(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
+        }
+    }
+}

test/HealthChecks/HealthCheckCollectorTokenHandlerTests.cs

@@ -0,0 +1,83 @@
+using System.Net;
+using Viper.Classes.HealthChecks;
+
+namespace Viper.test.HealthChecks
+{
+    public class HealthCheckCollectorTokenHandlerTests
+    {
+        [Fact]
+        public async Task SendAsync_StampsTokenHeader()
+        {
+            var (handler, recorder) = BuildHandler();
+            using var invoker = new HttpMessageInvoker(handler);
+            using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/health/detail");
+
+            using var response = await invoker.SendAsync(request, CancellationToken.None);
+
+            Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+            Assert.NotNull(recorder.LastRequest);
+            var values = recorder.LastRequest!.Headers.GetValues(HealthCheckCollectorAuth.HeaderName).ToList();
+            Assert.Single(values);
+            Assert.Equal(HealthCheckCollectorAuth.Token, values[0]);
+        }
+
+        [Fact]
+        public async Task SendAsync_ReplacesExistingTokenHeader()
+        {
+            var (handler, recorder) = BuildHandler();
+            using var invoker = new HttpMessageInvoker(handler);
+            using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/health/detail");
+            // Simulate a stale value left on the request before our handler runs.
+            request.Headers.Add(HealthCheckCollectorAuth.HeaderName, "stale-value");
+
+            await invoker.SendAsync(request, CancellationToken.None);
+
+            var values = recorder.LastRequest!.Headers.GetValues(HealthCheckCollectorAuth.HeaderName).ToList();
+            Assert.Single(values);
+            Assert.Equal(HealthCheckCollectorAuth.Token, values[0]);
+        }
+
+        [Fact]
+        public async Task SendAsync_ForwardsResponseFromInnerHandler()
+        {
+            var inner = new RecordingHandler(_ => new HttpResponseMessage(HttpStatusCode.Accepted));
+            var handler = new HealthCheckCollectorTokenHandler { InnerHandler = inner };
+            using var invoker = new HttpMessageInvoker(handler);
+            using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/health/detail");
+
+            using var response = await invoker.SendAsync(request, CancellationToken.None);
+
+            Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
+        }
+
+        private static (HealthCheckCollectorTokenHandler handler, RecordingHandler recorder) BuildHandler()
+        {
+            var inner = new RecordingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK));
+            var handler = new HealthCheckCollectorTokenHandler { InnerHandler = inner };
+            return (handler, inner);
+        }
+
+        /// <summary>
+        /// Inner DelegatingHandler that records the request it received and returns
+        /// a configurable response, so tests can assert what flowed downstream.
+        /// </summary>
+        private sealed class RecordingHandler : DelegatingHandler
+        {
+            private readonly Func<HttpRequestMessage, HttpResponseMessage> _respond;
+
+            public RecordingHandler(Func<HttpRequestMessage, HttpResponseMessage> respond)
+            {
+                _respond = respond;
+            }
+
+            public HttpRequestMessage? LastRequest { get; private set; }
+
+            protected override Task<HttpResponseMessage> SendAsync(
+                HttpRequestMessage request, CancellationToken cancellationToken)
+            {
+                LastRequest = request;
+                return Task.FromResult(_respond(request));
+            }
+        }
+    }
+}

web/Classes/HealthChecks/AdaptivePollingHealthCheck.cs

@@ -0,0 +1,81 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Viper.Classes.HealthChecks
+{
+    /// <summary>
+    /// Throttles an inner health check by caching its last result for a
+    /// status-dependent duration: Healthy results are reused for longer, while
+    /// Unhealthy/Degraded results refresh on a tighter cycle so recovery is
+    /// noticed quickly. When a cached result is returned, the original probe
+    /// timestamp is appended to the description so operators can tell how
+    /// stale the reading is.
+    /// </summary>
+    public class AdaptivePollingHealthCheck : IHealthCheck
+    {
+        private readonly IHealthCheck _inner;
+        private readonly TimeSpan _healthyCacheDuration;
+        private readonly TimeSpan _unhealthyCacheDuration;
+        private readonly SemaphoreSlim _semaphore = new(1, 1);
+        private DateTime _lastCheckTime;
+        private HealthCheckResult? _lastResult;
+
+        public AdaptivePollingHealthCheck(
+            IHealthCheck inner,
+            TimeSpan healthyCacheDuration,
+            TimeSpan unhealthyCacheDuration)
+        {
+            _inner = inner;
+            _healthyCacheDuration = healthyCacheDuration;
+            _unhealthyCacheDuration = unhealthyCacheDuration;
+        }
+
+        public async Task<HealthCheckResult> CheckHealthAsync(
+            HealthCheckContext context,
+            CancellationToken cancellationToken = default)
+        {
+            await _semaphore.WaitAsync(cancellationToken);
+            try
+            {
+                if (_lastResult.HasValue)
+                {
+                    // S6561: DateTime.Now used for elapsed-time calc. Accepted
+                    // because VIPER convention is DateTimeKind.Local and a
+                    // sub-hour DST skew only shifts one cache window.
+#pragma warning disable S6561
+                    var elapsed = DateTime.Now - _lastCheckTime;
+#pragma warning restore S6561
+                    var cacheDuration = _lastResult.Value.Status == HealthStatus.Healthy
+                        ? _healthyCacheDuration
+                        : _unhealthyCacheDuration;
+
+                    if (elapsed < cacheDuration)
+                    {
+                        return AppendTimestamp(_lastResult.Value, _lastCheckTime);
+                    }
+                }
+
+                var result = await _inner.CheckHealthAsync(context, cancellationToken);
+                _lastResult = result;
+                _lastCheckTime = DateTime.Now;
+                return result;
+            }
+            finally
+            {
+                _semaphore.Release();
+            }
+        }
+
+        private static HealthCheckResult AppendTimestamp(HealthCheckResult result, DateTime lastCheckedAt)
+        {
+            var stamp = $"Last checked: {lastCheckedAt:MMM d, h:mm tt}";
+            var description = string.IsNullOrWhiteSpace(result.Description)
+                ? stamp
+                : $"{result.Description}\n{stamp}";
+            return new HealthCheckResult(
+                result.Status,
+                description,
+                result.Exception,
+                result.Data);
+        }
+    }
+}

web/Classes/HealthChecks/AwsSsmHealthCheck.cs

@@ -0,0 +1,81 @@
+using Amazon;
+using Amazon.Runtime;
+using Amazon.SimpleSystemsManagement;
+using Amazon.SimpleSystemsManagement.Model;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Viper.Classes.HealthChecks
+{
+    /// <summary>
+    /// Verifies AWS SSM Parameter Store is reachable with the app's credentials.
+    /// Probes via GetParametersByPath against the same path prefix the
+    /// configuration loader uses (Program.cs .AddSystemsManager). This shares
+    /// the ssm:GetParametersByPath permission the app already needs, so a
+    /// least-privilege role passes here exactly when startup config load works -
+    /// DescribeParameters would require a separate ssm:DescribeParameters grant.
+    /// </summary>
+    public class AwsSsmHealthCheck : IHealthCheck
+    {
+        private readonly RegionEndpoint _region;
+        private readonly string _probePath;
+        private readonly bool _healthyWhenMissing;
+
+        /// <summary>
+        /// Constructs an AWS SSM Parameter Store reachability check.
+        /// </summary>
+        /// <param name="probePath">
+        /// SSM path prefix to probe (e.g. "/Shared"). Use a path the app's
+        /// configuration loader already reads so this check exercises the same
+        /// IAM permission.
+        /// </param>
+        /// <param name="region">
+        /// AWS region to query. Defaults to USWest1 when null.
+        /// </param>
+        /// <param name="healthyWhenMissing">
+        /// If true, missing credentials or client-side SDK errors return Healthy
+        /// with a "skipped" description. Use for Development where local machines
+        /// may not have AWS credentials configured.
+        /// </param>
+        public AwsSsmHealthCheck(
+            string probePath = "/Shared",
+            RegionEndpoint? region = null,
+            bool healthyWhenMissing = false)
+        {
+            _region = region ?? RegionEndpoint.USWest1;
+            _probePath = probePath;
+            _healthyWhenMissing = healthyWhenMissing;
+        }
+
+        public async Task<HealthCheckResult> CheckHealthAsync(
+            HealthCheckContext context,
+            CancellationToken cancellationToken = default)
+        {
+            try
+            {
+                using var client = new AmazonSimpleSystemsManagementClient(_region);
+                await client.GetParametersByPathAsync(
+                    new GetParametersByPathRequest
+                    {
+                        Path = _probePath,
+                        MaxResults = 1,
+                    },
+                    cancellationToken);
+                return HealthCheckResult.Healthy("AWS SSM reachable.");
+            }
+            catch (AmazonServiceException ex)
+            {
+                return _healthyWhenMissing
+                    ? HealthCheckResult.Healthy("AWS SSM not configured (skipped).")
+                    : HealthCheckResult.Unhealthy(
+                        $"AWS SSM unreachable: {ex.ErrorCode}: {ex.Message}", ex);
+            }
+            catch (AmazonClientException ex)
+            {
+                return _healthyWhenMissing
+                    ? HealthCheckResult.Healthy("AWS SSM not configured (skipped).")
+                    : HealthCheckResult.Unhealthy(
+                        $"AWS SSM client error: {ex.Message}", ex);
+            }
+        }
+    }
+}