Skip to content

Commit e3d52ee

Browse files
rlorenzorlorenzo
authored
Merge pull request #159 from ucdavis/VPR-141-health-check
VPR-141 feat(healthchecks): add /health endpoints and UI dashboard
2 parents 00d2eaa + 28e397f commit e3d52ee

18 files changed

Lines changed: 1460 additions & 10 deletions

‎JenkinsFile‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,27 @@ pipeline {
109109
networkShares()
110110
filecopy copyOfflineOperations, 'test', env.WORKSPACE
111111
filecopy copyOperations, 'test', env.WORKSPACE
112+
powershell label: 'Verify /health returns 200', script: '''
113+
$url = "https://secure-test.vetmed.ucdavis.edu/2/health"
114+
$maxAttempts = 15
115+
for ($i = 1; $i -le $maxAttempts; $i++) {
116+
# 2s for the first few attempts then 4s - absorbs IIS app-pool warm-up
117+
$delay = if ($i -le 5) { 2 } else { 4 }
118+
try {
119+
$response = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 10
120+
if ($response.StatusCode -eq 200) {
121+
Write-Host "Attempt ${i}: $url returned 200 OK"
122+
exit 0
123+
}
124+
Write-Host "Attempt ${i}: $url returned $($response.StatusCode)"
125+
} catch {
126+
Write-Host "Attempt ${i}: $($_.Exception.Message)"
127+
}
128+
if ($i -lt $maxAttempts) { Start-Sleep -Seconds $delay }
129+
}
130+
Write-Error "Health check at $url failed after $maxAttempts attempts"
131+
exit 1
132+
'''
112133
}
113134
}
114135
stage('Deploy to prod') {
@@ -122,6 +143,26 @@ pipeline {
122143
networkShares()
123144
filecopy copyOfflineOperations, 'prod', env.WORKSPACE
124145
filecopy copyOperations, 'prod', env.WORKSPACE
146+
powershell label: 'Verify /health returns 200', script: '''
147+
$url = "https://secure.vetmed.ucdavis.edu/2/health"
148+
$maxAttempts = 15
149+
for ($i = 1; $i -le $maxAttempts; $i++) {
150+
$delay = if ($i -le 5) { 2 } else { 4 }
151+
try {
152+
$response = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec 10
153+
if ($response.StatusCode -eq 200) {
154+
Write-Host "Attempt ${i}: $url returned 200 OK"
155+
exit 0
156+
}
157+
Write-Host "Attempt ${i}: $url returned $($response.StatusCode)"
158+
} catch {
159+
Write-Host "Attempt ${i}: $($_.Exception.Message)"
160+
}
161+
if ($i -lt $maxAttempts) { Start-Sleep -Seconds $delay }
162+
}
163+
Write-Error "Health check at $url failed after $maxAttempts attempts"
164+
exit 1
165+
'''
125166
}
126167
}
127168
}

‎test/HealthChecks/HealthCheckCollectorAuthTests.cs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using Microsoft.AspNetCore.Http;
2+
using Viper.Classes.HealthChecks;
3+
4+
namespace Viper.test.HealthChecks
5+
{
6+
public class HealthCheckCollectorAuthTests
7+
{
8+
[Fact]
9+
public void Matches_NullProvided_ReturnsFalse()
10+
{
11+
Assert.False(HealthCheckCollectorAuth.Matches(null));
12+
}
13+
14+
[Fact]
15+
public void Matches_EmptyProvided_ReturnsFalse()
16+
{
17+
Assert.False(HealthCheckCollectorAuth.Matches(string.Empty));
18+
}
19+
20+
[Fact]
21+
public void Matches_DifferentLength_ReturnsFalse()
22+
{
23+
Assert.False(HealthCheckCollectorAuth.Matches("short"));
24+
}
25+
26+
[Fact]
27+
public void Matches_SameLengthDifferentValue_ReturnsFalse()
28+
{
29+
// Same length as the real token (32 hex chars from Guid.ToString("N")) but different bytes.
30+
var sameLengthFake = new string('a', HealthCheckCollectorAuth.Token.Length);
31+
Assert.False(HealthCheckCollectorAuth.Matches(sameLengthFake));
32+
}
33+
34+
[Fact]
35+
public void Matches_RealToken_ReturnsTrue()
36+
{
37+
Assert.True(HealthCheckCollectorAuth.Matches(HealthCheckCollectorAuth.Token));
38+
}
39+
40+
[Fact]
41+
public void IsCollectorRequest_HeaderAbsent_ReturnsFalse()
42+
{
43+
var ctx = new DefaultHttpContext();
44+
Assert.False(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
45+
}
46+
47+
[Fact]
48+
public void IsCollectorRequest_HeaderEmpty_ReturnsFalse()
49+
{
50+
var ctx = new DefaultHttpContext();
51+
ctx.Request.Headers[HealthCheckCollectorAuth.HeaderName] = string.Empty;
52+
Assert.False(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
53+
}
54+
55+
[Fact]
56+
public void IsCollectorRequest_WrongToken_ReturnsFalse()
57+
{
58+
var ctx = new DefaultHttpContext();
59+
ctx.Request.Headers[HealthCheckCollectorAuth.HeaderName] = "not-the-real-token";
60+
Assert.False(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
61+
}
62+
63+
[Fact]
64+
public void IsCollectorRequest_ValidToken_ReturnsTrue()
65+
{
66+
var ctx = new DefaultHttpContext();
67+
ctx.Request.Headers[HealthCheckCollectorAuth.HeaderName] = HealthCheckCollectorAuth.Token;
68+
Assert.True(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
69+
}
70+
71+
[Fact]
72+
public void IsCollectorRequest_HeaderNameIsCaseInsensitive()
73+
{
74+
// ASP.NET request headers are case-insensitive; the filter must accept any casing
75+
// a proxy might apply.
76+
var ctx = new DefaultHttpContext();
77+
ctx.Request.Headers["x-health-collector-token"] = HealthCheckCollectorAuth.Token;
78+
Assert.True(HealthCheckCollectorAuth.IsCollectorRequest(ctx));
79+
}
80+
}
81+
}

‎test/HealthChecks/HealthCheckCollectorTokenHandlerTests.cs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
using System.Net;
2+
using Viper.Classes.HealthChecks;
3+
4+
namespace Viper.test.HealthChecks
5+
{
6+
public class HealthCheckCollectorTokenHandlerTests
7+
{
8+
[Fact]
9+
public async Task SendAsync_StampsTokenHeader()
10+
{
11+
var (handler, recorder) = BuildHandler();
12+
using var invoker = new HttpMessageInvoker(handler);
13+
using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/health/detail");
14+
15+
using var response = await invoker.SendAsync(request, CancellationToken.None);
16+
17+
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
18+
Assert.NotNull(recorder.LastRequest);
19+
var values = recorder.LastRequest!.Headers.GetValues(HealthCheckCollectorAuth.HeaderName).ToList();
20+
Assert.Single(values);
21+
Assert.Equal(HealthCheckCollectorAuth.Token, values[0]);
22+
}
23+
24+
[Fact]
25+
public async Task SendAsync_ReplacesExistingTokenHeader()
26+
{
27+
var (handler, recorder) = BuildHandler();
28+
using var invoker = new HttpMessageInvoker(handler);
29+
using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/health/detail");
30+
// Simulate a stale value left on the request before our handler runs.
31+
request.Headers.Add(HealthCheckCollectorAuth.HeaderName, "stale-value");
32+
33+
await invoker.SendAsync(request, CancellationToken.None);
34+
35+
var values = recorder.LastRequest!.Headers.GetValues(HealthCheckCollectorAuth.HeaderName).ToList();
36+
Assert.Single(values);
37+
Assert.Equal(HealthCheckCollectorAuth.Token, values[0]);
38+
}
39+
40+
[Fact]
41+
public async Task SendAsync_ForwardsResponseFromInnerHandler()
42+
{
43+
var inner = new RecordingHandler(_ => new HttpResponseMessage(HttpStatusCode.Accepted));
44+
var handler = new HealthCheckCollectorTokenHandler { InnerHandler = inner };
45+
using var invoker = new HttpMessageInvoker(handler);
46+
using var request = new HttpRequestMessage(HttpMethod.Get, "https://example.test/health/detail");
47+
48+
using var response = await invoker.SendAsync(request, CancellationToken.None);
49+
50+
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
51+
}
52+
53+
private static (HealthCheckCollectorTokenHandler handler, RecordingHandler recorder) BuildHandler()
54+
{
55+
var inner = new RecordingHandler(_ => new HttpResponseMessage(HttpStatusCode.OK));
56+
var handler = new HealthCheckCollectorTokenHandler { InnerHandler = inner };
57+
return (handler, inner);
58+
}
59+
60+
/// <summary>
61+
/// Inner DelegatingHandler that records the request it received and returns
62+
/// a configurable response, so tests can assert what flowed downstream.
63+
/// </summary>
64+
private sealed class RecordingHandler : DelegatingHandler
65+
{
66+
private readonly Func<HttpRequestMessage, HttpResponseMessage> _respond;
67+
68+
public RecordingHandler(Func<HttpRequestMessage, HttpResponseMessage> respond)
69+
{
70+
_respond = respond;
71+
}
72+
73+
public HttpRequestMessage? LastRequest { get; private set; }
74+
75+
protected override Task<HttpResponseMessage> SendAsync(
76+
HttpRequestMessage request, CancellationToken cancellationToken)
77+
{
78+
LastRequest = request;
79+
return Task.FromResult(_respond(request));
80+
}
81+
}
82+
}
83+
}

‎web/Classes/CloudflareNetworks.cs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
namespace Viper.Classes
2+
{
3+
/// <summary>
4+
/// Cloudflare's published IPv4/IPv6 networks, used to mark CF as a known
5+
/// proxy in ForwardedHeadersOptions. Fetched from cloudflare.com at startup
6+
/// so we automatically pick up rotations; falls back to a hardcoded snapshot
7+
/// when the fetch fails (CF outage during deploy, sandboxed network, etc).
8+
/// </summary>
9+
public static class CloudflareNetworks
10+
{
11+
// Snapshot of https://www.cloudflare.com/ips/ - only used when the
12+
// runtime fetch fails. Refresh occasionally if logs show this falling
13+
// through and current CF IPs aren't in the list.
14+
private static readonly string[] HardcodedFallback =
15+
[
16+
"173.245.48.0/20",
17+
"103.21.244.0/22",
18+
"103.22.200.0/22",
19+
"103.31.4.0/22",
20+
"141.101.64.0/18",
21+
"108.162.192.0/18",
22+
"190.93.240.0/20",
23+
"188.114.96.0/20",
24+
"197.234.240.0/22",
25+
"198.41.128.0/17",
26+
"162.158.0.0/15",
27+
"104.16.0.0/13",
28+
"104.24.0.0/14",
29+
"172.64.0.0/13",
30+
"131.0.72.0/22",
31+
"2400:cb00::/32",
32+
"2606:4700::/32",
33+
"2803:f800::/32",
34+
"2405:b500::/32",
35+
"2405:8100::/32",
36+
"2a06:98c0::/29",
37+
"2c0f:f248::/32",
38+
];
39+
40+
public static IReadOnlyList<string> FetchOrFallback(NLog.Logger logger)
41+
{
42+
try
43+
{
44+
// Startup-only call, not in a hot path - the socket-exhaustion concern
45+
// behind ShortLivedHttpClient doesn't apply here.
46+
// ReSharper disable once ShortLivedHttpClient
47+
using var http = new HttpClient();
48+
http.Timeout = TimeSpan.FromSeconds(5);
49+
var v4 = http.GetStringAsync("https://www.cloudflare.com/ips-v4/").GetAwaiter().GetResult();
50+
var v6 = http.GetStringAsync("https://www.cloudflare.com/ips-v6/").GetAwaiter().GetResult();
51+
var cidrs = (v4 + "\n" + v6)
52+
.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
53+
logger.Info("Fetched {Count} Cloudflare networks from cloudflare.com", cidrs.Length);
54+
return cidrs;
55+
}
56+
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
57+
{
58+
logger.Warn(ex, "Failed to fetch Cloudflare IP ranges; using hardcoded fallback ({Count} entries)", HardcodedFallback.Length);
59+
return HardcodedFallback;
60+
}
61+
}
62+
}
63+
}

‎web/Classes/HealthChecks/AdaptivePollingHealthCheck.cs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
using Microsoft.Extensions.Diagnostics.HealthChecks;
2+
3+
namespace Viper.Classes.HealthChecks
4+
{
5+
/// <summary>
6+
/// Throttles an inner health check by caching its last result for a
7+
/// status-dependent duration: Healthy results are reused for longer, while
8+
/// Unhealthy/Degraded results refresh on a tighter cycle so recovery is
9+
/// noticed quickly. When a cached result is returned, the original probe
10+
/// timestamp is appended to the description so operators can tell how
11+
/// stale the reading is.
12+
/// </summary>
13+
public class AdaptivePollingHealthCheck : IHealthCheck
14+
{
15+
private readonly IHealthCheck _inner;
16+
private readonly TimeSpan _healthyCacheDuration;
17+
private readonly TimeSpan _unhealthyCacheDuration;
18+
private readonly SemaphoreSlim _semaphore = new(1, 1);
19+
private DateTime _lastCheckTime;
20+
private HealthCheckResult? _lastResult;
21+
22+
public AdaptivePollingHealthCheck(
23+
IHealthCheck inner,
24+
TimeSpan healthyCacheDuration,
25+
TimeSpan unhealthyCacheDuration)
26+
{
27+
_inner = inner;
28+
_healthyCacheDuration = healthyCacheDuration;
29+
_unhealthyCacheDuration = unhealthyCacheDuration;
30+
}
31+
32+
public async Task<HealthCheckResult> CheckHealthAsync(
33+
HealthCheckContext context,
34+
CancellationToken cancellationToken = default)
35+
{
36+
await _semaphore.WaitAsync(cancellationToken);
37+
try
38+
{
39+
if (_lastResult.HasValue)
40+
{
41+
// S6561: DateTime.Now used for elapsed-time calc. Accepted
42+
// because VIPER convention is DateTimeKind.Local and a
43+
// sub-hour DST skew only shifts one cache window.
44+
#pragma warning disable S6561
45+
var elapsed = DateTime.Now - _lastCheckTime;
46+
#pragma warning restore S6561
47+
var cacheDuration = _lastResult.Value.Status == HealthStatus.Healthy
48+
? _healthyCacheDuration
49+
: _unhealthyCacheDuration;
50+
51+
if (elapsed < cacheDuration)
52+
{
53+
return AppendTimestamp(_lastResult.Value, _lastCheckTime);
54+
}
55+
}
56+
57+
var result = await _inner.CheckHealthAsync(context, cancellationToken);
58+
_lastResult = result;
59+
_lastCheckTime = DateTime.Now;
60+
return result;
61+
}
62+
finally
63+
{
64+
_semaphore.Release();
65+
}
66+
}
67+
68+
private static HealthCheckResult AppendTimestamp(HealthCheckResult result, DateTime lastCheckedAt)
69+
{
70+
var stamp = $"Last checked: {lastCheckedAt:MMM d, h:mm tt}";
71+
var description = string.IsNullOrWhiteSpace(result.Description)
72+
? stamp
73+
: $"{result.Description}\n{stamp}";
74+
return new HealthCheckResult(
75+
result.Status,
76+
description,
77+
result.Exception,
78+
result.Data);
79+
}
80+
}
81+
}

0 commit comments

Comments
 (0)