From 3c900138c9e7585f3667c5b135f8ecf093fae75c Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Sun, 19 Jul 2026 13:43:14 -0500 Subject: [PATCH 01/23] Show UniFi display name on 2D/3D maps and Client Performance The maps (DiscoveredClient via stat/sta) and Client Performance's online identify path only had v1 name/hostname, so name-less clients rendered as a raw MAC while Client Stats showed UniFi's friendly display_name (v2 active-clients). Plumb that display_name into both as the top of the fallback chain (DisplayName > Name > Hostname > MAC) via a dedicated per-connection labels-only cache (5 min TTL) that leaves the real-time active-clients call uncached. --- .../ClientDisplayNameCache.cs | 64 +++++++++++++++++++ src/NetworkOptimizer.UniFi/UniFiDiscovery.cs | 13 ++++ .../Services/ClientDashboardService.cs | 21 ++++-- .../Services/LanFlowMap/LanFlowMapService.cs | 8 ++- 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 src/NetworkOptimizer.UniFi/ClientDisplayNameCache.cs diff --git a/src/NetworkOptimizer.UniFi/ClientDisplayNameCache.cs b/src/NetworkOptimizer.UniFi/ClientDisplayNameCache.cs new file mode 100644 index 000000000..06ca79d10 --- /dev/null +++ b/src/NetworkOptimizer.UniFi/ClientDisplayNameCache.cs @@ -0,0 +1,64 @@ +using System.Runtime.CompilerServices; + +namespace NetworkOptimizer.UniFi; + +/// +/// Per-connection (per-site) cache of the UniFi v2 active-clients display_name - the +/// system-selected friendly device name the console shows (e.g. "[IoT] Tiny Home - Plug", or a +/// fingerprint-derived name like "Apple TV" for a device the user never named). Exposed as a +/// lower-cased-MAC -> name lookup and refreshed at most once every 5 minutes. +/// +/// This deliberately lives OUTSIDE : the real-time +/// stays uncached, and only this label-only +/// projection is cached - so callers that need live client state are never served stale data. +/// Label-only consumers (the 2D/3D LAN flow maps and Client Performance) use this instead of +/// hitting the v2 endpoint on every request. Entries are keyed weakly by +/// instance, so a site's cache is evicted when its connection is torn down. +/// +public static class ClientDisplayNameCache +{ + private static readonly TimeSpan Ttl = TimeSpan.FromMinutes(5); + + private sealed class Entry + { + public readonly SemaphoreSlim Gate = new(1, 1); + public IReadOnlyDictionary Map = + new Dictionary(StringComparer.OrdinalIgnoreCase); + public DateTime FetchedUtc = DateTime.MinValue; + } + + private static readonly ConditionalWeakTable Cache = new(); + + /// + /// Returns a lower-cased-MAC -> display_name lookup for the given connection, refreshing + /// from the v2 active-clients endpoint at most once per 5 minutes. Clients without a display name + /// are omitted, so callers keep their own downstream fallback (name > hostname > MAC) for those. + /// + public static async Task> GetAsync( + UniFiApiClient client, CancellationToken cancellationToken = default) + { + var entry = Cache.GetOrCreateValue(client); + if (DateTime.UtcNow - entry.FetchedUtc < Ttl) + return entry.Map; + + await entry.Gate.WaitAsync(cancellationToken); + try + { + // Re-check under the lock: another caller may have refreshed while we waited. + if (DateTime.UtcNow - entry.FetchedUtc < Ttl) + return entry.Map; + + var active = await client.GetActiveClientsAsync(cancellationToken); + entry.Map = active + .Where(c => !string.IsNullOrEmpty(c.DisplayName) && !string.IsNullOrEmpty(c.Mac)) + .GroupBy(c => c.Mac.ToLowerInvariant()) + .ToDictionary(g => g.Key, g => g.First().DisplayName!, StringComparer.OrdinalIgnoreCase); + entry.FetchedUtc = DateTime.UtcNow; + return entry.Map; + } + finally + { + entry.Gate.Release(); + } + } +} diff --git a/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs b/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs index 4d4fe1156..d88c032c7 100644 --- a/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs +++ b/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs @@ -360,6 +360,11 @@ public async Task> DiscoverClientsAsync(CancellationToken } } + // UniFi's friendly display name (fingerprint / system-selected) lives only on the v2 + // active-clients endpoint, not on stat/sta. Pull it (cached 5 min, labels only) so the + // 2D/3D maps show the same name as Wi-Fi Optimizer - Client Stats instead of a raw MAC. + var displayNames = await ClientDisplayNameCache.GetAsync(_apiClient, cancellationToken); + var discoveredClients = clients.Select(c => { // Use stat/sta BestIp (ip > last_ip > fixed_ip), then active clients endpoint (UX/UX7 bug workaround) @@ -375,6 +380,7 @@ public async Task> DiscoverClientsAsync(CancellationToken Mac = c.Mac, Hostname = c.Hostname, Name = c.Name, + DisplayName = displayNames.TryGetValue(c.Mac.ToLowerInvariant(), out var dn) ? dn : string.Empty, IpAddress = ipAddress ?? string.Empty, Network = c.Network, NetworkId = c.NetworkId, @@ -872,6 +878,13 @@ public class DiscoveredClient public string Mac { get; set; } = string.Empty; public string Hostname { get; set; } = string.Empty; public string Name { get; set; } = string.Empty; + /// + /// UniFi's system-selected friendly device name from the v2 active-clients endpoint + /// (the "device print name" the console shows, e.g. a fingerprint-derived "Apple TV" + /// for an unnamed device). Populated for currently-active clients; empty when unknown. + /// Prefer this over / for display labels. + /// + public string DisplayName { get; set; } = string.Empty; public string IpAddress { get; set; } = string.Empty; public string Network { get; set; } = string.Empty; public string NetworkId { get; set; } = string.Empty; diff --git a/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs b/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs index c5e4814aa..c737a88dd 100644 --- a/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs +++ b/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs @@ -92,11 +92,15 @@ public async Task> GetSelectableClientsAsync() try { var clients = await _connectionService.Client.GetClientsAsync(); + // Overlay UniFi's friendly display name (v2 active-clients, cached 5 min) so the + // picker matches the name shown on the page and in Client Stats. + var displayNames = await ClientDisplayNameCache.GetAsync(_connectionService.Client); return (clients ?? new List()) .Where(c => !string.IsNullOrEmpty(c.BestIp)) .Select(c => new SelectableClient( c.BestIp!, - !string.IsNullOrWhiteSpace(c.Name) ? c.Name + displayNames.TryGetValue(c.Mac.ToLowerInvariant(), out var dn) ? dn + : !string.IsNullOrWhiteSpace(c.Name) ? c.Name : !string.IsNullOrWhiteSpace(c.Hostname) ? c.Hostname : c.BestIp!, c.IsWired)) .OrderBy(c => c.Name, StringComparer.OrdinalIgnoreCase) @@ -164,7 +168,12 @@ public async Task> GetSelectableClientsAsync() _offlineIdentityCache.TryRemove(clientIp, out _); _ipToMacCache[clientIp] = client.Mac; - var identity = MapClientToIdentity(client); + // UniFi's friendly display name is only on the v2 active-clients endpoint, not + // stat/sta; pull it (cached 5 min, labels only) so an unnamed device shows the + // same name here as in Client Stats instead of a raw MAC. + var displayNames = await ClientDisplayNameCache.GetAsync(_connectionService.Client); + displayNames.TryGetValue(client.Mac.ToLowerInvariant(), out var displayName); + var identity = MapClientToIdentity(client, displayName); // Try WiFiman endpoint for more-realtime signal data, overlay on top of stat/sta await OverlayWiFiManDataAsync(identity, clientIp); @@ -940,12 +949,16 @@ private async Task StoreSignalLogAsync( } } - private ClientIdentity MapClientToIdentity(UniFiClientResponse client) + private ClientIdentity MapClientToIdentity(UniFiClientResponse client, string? displayName = null) { return new ClientIdentity { Mac = client.Mac, - Name = !string.IsNullOrEmpty(client.Name) ? client.Name : null, + // Prefer UniFi's system-selected display name (v2 active-clients, e.g. a + // fingerprint-derived "Apple TV") over the raw stat/sta name, matching Client + // Stats and the offline-identity path so an unnamed device never shows as a MAC. + Name = !string.IsNullOrEmpty(displayName) ? displayName + : !string.IsNullOrEmpty(client.Name) ? client.Name : null, Hostname = !string.IsNullOrEmpty(client.Hostname) ? client.Hostname : null, Ip = client.Ip, IsWired = client.IsWired, diff --git a/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs b/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs index e392be5bc..72a41f401 100644 --- a/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs +++ b/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs @@ -2026,12 +2026,14 @@ private static string NormalizeMac(string? mac) => string.IsNullOrEmpty(mac) ? string.Empty : mac.ToLowerInvariant().Replace("-", ":"); /// - /// Client label fallback chain that matches what the audit / port-security - /// analyzers use: user-set Name > device-reported Hostname > MAC. Keeps the 3D - /// map labels consistent with the rest of the UI. + /// Client label fallback chain matching Wi-Fi Optimizer - Client Stats: UniFi's + /// system-selected DisplayName (v2 active-clients) > user-set Name > device-reported + /// Hostname > MAC. The DisplayName step keeps name-less clients from rendering as a + /// raw MAC on the 2D/3D maps when the console has a friendly/fingerprint name for them. /// private static string ResolveClientLabel(NetworkOptimizer.UniFi.DiscoveredClient c) { + if (!string.IsNullOrWhiteSpace(c.DisplayName)) return c.DisplayName; if (!string.IsNullOrWhiteSpace(c.Name)) return c.Name; if (!string.IsNullOrWhiteSpace(c.Hostname)) return c.Hostname; return string.IsNullOrEmpty(c.Mac) ? "unknown" : c.Mac; From 2deef190b40a09720871b7ac560e9773a8fd35da Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:24:31 -0500 Subject: [PATCH 02/23] Fix duplicate Health Issues entries and a spurious console-connection error (#1026) * Fix intermittent duplicate Wi-Fi Optimizer Health Issues entries GetSiteHealthScoreAsync assigned the fresh score to the shared _cachedHealthScore field and then mutated that field in place (MLO/6 GHz checks + EvaluateRules). With no synchronization, an initial load racing the Overview auto-refresh could interleave so both calls appended their findings onto the same score object, doubling every Health Issues entry (the '(2)' badges). Serialize refreshes with a semaphore and build the score in a local, publishing it to the field only once fully assembled. * Don't surface a disposed-client login as a console-connection error On reconnect (e.g. a restart while an agent-tunnel site's console is coming back up) the UniFiApiClient and its _authLock can be disposed while a LoginAsync is still in flight. The finally's _authLock.Release() then threw ObjectDisposedException, which bubbled up and showed as a 'Cannot access a disposed object' error in the Wi-Fi Optimizer console-connection warning box. Guard just that Release so a disposed-mid-login degrades to the normal transient retry instead of a scary error. --- src/NetworkOptimizer.UniFi/UniFiApiClient.cs | 7 +- .../Services/WiFiOptimizerService.cs | 189 ++++++++++-------- 2 files changed, 110 insertions(+), 86 deletions(-) diff --git a/src/NetworkOptimizer.UniFi/UniFiApiClient.cs b/src/NetworkOptimizer.UniFi/UniFiApiClient.cs index 665415dcd..86ba7b587 100644 --- a/src/NetworkOptimizer.UniFi/UniFiApiClient.cs +++ b/src/NetworkOptimizer.UniFi/UniFiApiClient.cs @@ -361,7 +361,12 @@ public async Task LoginAsync(CancellationToken cancellationToken = default } finally { - _authLock.Release(); + // On a reconnect the client (and its _authLock) can be disposed while this login + // is still in flight; Release() would then throw ObjectDisposedException out of the + // finally and surface as a scary "Cannot access a disposed object" console-connection + // error. There's nothing to release on a disposed semaphore, so ignore just that case. + try { _authLock.Release(); } + catch (ObjectDisposedException) { } } } diff --git a/src/NetworkOptimizer.Web/Services/WiFiOptimizerService.cs b/src/NetworkOptimizer.Web/Services/WiFiOptimizerService.cs index a071210ff..29bd8a1d1 100644 --- a/src/NetworkOptimizer.Web/Services/WiFiOptimizerService.cs +++ b/src/NetworkOptimizer.Web/Services/WiFiOptimizerService.cs @@ -47,6 +47,10 @@ public class WiFiOptimizerService private SiteHealthScore? _cachedHealthScore; private DateTimeOffset _lastRefresh = DateTimeOffset.MinValue; private readonly TimeSpan _cacheExpiry = TimeSpan.FromSeconds(30); + // Serializes health-score refreshes. Overlapping calls (initial load racing the Overview + // auto-refresh) otherwise interleave on the shared _cachedHealthScore field and append + // each finding twice, showing duplicate Health Issues entries. + private readonly SemaphoreSlim _healthScoreLock = new(1, 1); public WiFiOptimizerService( UniFiConnectionService connectionService, @@ -109,15 +113,25 @@ private UniFiLiveDataProvider CreateProvider() return _cachedHealthScore; } + await _healthScoreLock.WaitAsync(); try { + // Re-check under the lock: a concurrent caller may have refreshed while we waited. + if (!forceRefresh && _cachedHealthScore != null && DateTimeOffset.UtcNow - _lastRefresh < _cacheExpiry) + { + return _cachedHealthScore; + } + await RefreshDataAsync(); if (_cachedAps == null || _cachedClients == null) { return null; } - _cachedHealthScore = _healthScorer.Calculate(_cachedAps, _cachedClients, _cachedRoamingData); + // Build into a local score and only publish it to the shared field once fully + // assembled - never mutate _cachedHealthScore in place, or a caller could observe + // (or a racing refresh could double up) a half-populated Issues list. + var score = _healthScorer.Calculate(_cachedAps, _cachedClients, _cachedRoamingData); // Only consider online APs for additional issue checks var onlineAps = _cachedAps.Where(ap => ap.IsOnline).ToList(); @@ -127,7 +141,7 @@ private UniFiLiveDataProvider CreateProvider() var hasMloEnabledWlan = _cachedWlanConfigs?.Any(w => w.Enabled && w.MloEnabled) == true; if (hasWifi7Aps && hasMloEnabledWlan) { - _cachedHealthScore.Issues.Add(new HealthIssue + score.Issues.Add(new HealthIssue { Severity = HealthIssueSeverity.Info, Dimensions = { HealthDimension.AirtimeEfficiency }, @@ -143,7 +157,7 @@ private UniFiLiveDataProvider CreateProvider() if (hasAps6GHz && !hasWlan6GHz) { var aps6GHzCount = onlineAps.Count(ap => ap.Radios.Any(r => r.Band == RadioBand.Band6GHz)); - _cachedHealthScore.Issues.Add(new HealthIssue + score.Issues.Add(new HealthIssue { Severity = HealthIssueSeverity.Info, Dimensions = { HealthDimension.ChannelHealth, HealthDimension.AirtimeEfficiency }, @@ -157,16 +171,21 @@ private UniFiLiveDataProvider CreateProvider() if (_cachedWlanConfigs != null && _cachedNetworks != null) { var context = await BuildOptimizerContextAsync(onlineAps, _cachedClients, _cachedWlanConfigs, _cachedNetworks); - _optimizerEngine.EvaluateRules(_cachedHealthScore, context); + _optimizerEngine.EvaluateRules(score, context); } - return _cachedHealthScore; + _cachedHealthScore = score; + return score; } catch (Exception ex) { _logger.LogError(ex, "Failed to calculate site health score"); return null; } + finally + { + _healthScoreLock.Release(); + } } /// @@ -1352,106 +1371,106 @@ private sealed class HistoricalChannelContext // fall through with whatever stress was built plus the memory/soak sections below. try { - foreach (var (mac, metrics, recentMetrics, events) in allResults) - { - if (metrics.Count == 0) continue; - var macLower = mac.ToLowerInvariant(); - - // Find the current channel for each band from the AP snapshot - var ap = onlineAps.First(a => a.Mac.Equals(mac, StringComparison.OrdinalIgnoreCase)); - - foreach (var band in bands) + foreach (var (mac, metrics, recentMetrics, events) in allResults) { - var radio = ap.Radios.FirstOrDefault(r => r.Band == band && r.Channel.HasValue); - if (radio == null) continue; - - // Build channel timeline from change events (sorted chronologically) - var bandEvents = events - .Where(e => e.Band == band) - .OrderBy(e => e.Timestamp) - .ToList(); + if (metrics.Count == 0) continue; + var macLower = mac.ToLowerInvariant(); - _logger.LogDebug("[ChannelRec] {ApName} {Band}: {EventCount} channel events, current=ch{CurrentCh}, events=[{Events}]", - ap.Name, band, bandEvents.Count, radio.Channel, - string.Join(", ", bandEvents.Select(e => $"{e.Timestamp:MM/dd} ch{e.PreviousChannel}→ch{e.NewChannel}"))); + // Find the current channel for each band from the AP snapshot + var ap = onlineAps.First(a => a.Mac.Equals(mac, StringComparison.OrdinalIgnoreCase)); - // 7-day hourly: average per channel - var channelMetrics = new Dictionary>(); - foreach (var metric in metrics) + foreach (var band in bands) { - if (!metric.ByBand.TryGetValue(band, out var bandData) || - !bandData.ChannelUtilization.HasValue) - continue; - - var channel = ChannelMemoryHelper.GetChannelAtTime(metric.Timestamp, bandEvents, radio.Channel!.Value); - // An event parsed without PREVIOUS_CHANNEL yields channel 0 for samples - // predating it - not a real channel, don't build stress for it. - if (channel <= 0) continue; - if (!channelMetrics.ContainsKey(channel)) - channelMetrics[channel] = new List<(double, double, double)>(); - channelMetrics[channel].Add(( - bandData.ChannelUtilization ?? 0, - bandData.Interference ?? 0, - bandData.TxRetryPct ?? 0)); - } + var radio = ap.Radios.FirstOrDefault(r => r.Band == band && r.Channel.HasValue); + if (radio == null) continue; - // 1-day 5-min: average for current channel only (higher resolution recent data) - var recentCurrentChannel = new List<(double Util, double Interf, double TxRetry)>(); - foreach (var metric in recentMetrics) - { - if (!metric.ByBand.TryGetValue(band, out var bandData) || - !bandData.ChannelUtilization.HasValue) - continue; + // Build channel timeline from change events (sorted chronologically) + var bandEvents = events + .Where(e => e.Band == band) + .OrderBy(e => e.Timestamp) + .ToList(); - var channel = ChannelMemoryHelper.GetChannelAtTime(metric.Timestamp, bandEvents, radio.Channel!.Value); - if (channel == radio.Channel!.Value) + _logger.LogDebug("[ChannelRec] {ApName} {Band}: {EventCount} channel events, current=ch{CurrentCh}, events=[{Events}]", + ap.Name, band, bandEvents.Count, radio.Channel, + string.Join(", ", bandEvents.Select(e => $"{e.Timestamp:MM/dd} ch{e.PreviousChannel}→ch{e.NewChannel}"))); + + // 7-day hourly: average per channel + var channelMetrics = new Dictionary>(); + foreach (var metric in metrics) { - recentCurrentChannel.Add(( + if (!metric.ByBand.TryGetValue(band, out var bandData) || + !bandData.ChannelUtilization.HasValue) + continue; + + var channel = ChannelMemoryHelper.GetChannelAtTime(metric.Timestamp, bandEvents, radio.Channel!.Value); + // An event parsed without PREVIOUS_CHANNEL yields channel 0 for samples + // predating it - not a real channel, don't build stress for it. + if (channel <= 0) continue; + if (!channelMetrics.ContainsKey(channel)) + channelMetrics[channel] = new List<(double, double, double)>(); + channelMetrics[channel].Add(( bandData.ChannelUtilization ?? 0, bandData.Interference ?? 0, bandData.TxRetryPct ?? 0)); } - } - if (channelMetrics.Count > 0) - { - var perChannel = new Dictionary(); - foreach (var (ch, dataPoints) in channelMetrics) + // 1-day 5-min: average for current channel only (higher resolution recent data) + var recentCurrentChannel = new List<(double Util, double Interf, double TxRetry)>(); + foreach (var metric in recentMetrics) { - var avg = ( - dataPoints.Average(d => d.Util), - dataPoints.Average(d => d.Interf), - dataPoints.Average(d => d.TxRetry)); - - // For current channel: use max of 7-day avg and 1-day avg - // so recent deterioration isn't diluted by older data - if (ch == radio.Channel!.Value && recentCurrentChannel.Count > 0) + if (!metric.ByBand.TryGetValue(band, out var bandData) || + !bandData.ChannelUtilization.HasValue) + continue; + + var channel = ChannelMemoryHelper.GetChannelAtTime(metric.Timestamp, bandEvents, radio.Channel!.Value); + if (channel == radio.Channel!.Value) { - var recentAvg = ( - recentCurrentChannel.Average(d => d.Util), - recentCurrentChannel.Average(d => d.Interf), - recentCurrentChannel.Average(d => d.TxRetry)); - - avg = ( - Math.Max(avg.Item1, recentAvg.Item1), - Math.Max(avg.Item2, recentAvg.Item2), - Math.Max(avg.Item3, recentAvg.Item3)); - - _logger.LogDebug("[ChannelRec] {ApName} {Band} ch{Ch}: 7d avg u={U7:F1}% i={I7:F1}% tx={T7:F1}%, " + - "1d avg u={U1:F1}% i={I1:F1}% tx={T1:F1}% ({Count} samples), using max", - ap.Name, band, ch, - dataPoints.Average(d => d.Util), dataPoints.Average(d => d.Interf), dataPoints.Average(d => d.TxRetry), - recentAvg.Item1, recentAvg.Item2, recentAvg.Item3, recentCurrentChannel.Count); + recentCurrentChannel.Add(( + bandData.ChannelUtilization ?? 0, + bandData.Interference ?? 0, + bandData.TxRetryPct ?? 0)); } + } - perChannel[ch] = avg; + if (channelMetrics.Count > 0) + { + var perChannel = new Dictionary(); + foreach (var (ch, dataPoints) in channelMetrics) + { + var avg = ( + dataPoints.Average(d => d.Util), + dataPoints.Average(d => d.Interf), + dataPoints.Average(d => d.TxRetry)); + + // For current channel: use max of 7-day avg and 1-day avg + // so recent deterioration isn't diluted by older data + if (ch == radio.Channel!.Value && recentCurrentChannel.Count > 0) + { + var recentAvg = ( + recentCurrentChannel.Average(d => d.Util), + recentCurrentChannel.Average(d => d.Interf), + recentCurrentChannel.Average(d => d.TxRetry)); + + avg = ( + Math.Max(avg.Item1, recentAvg.Item1), + Math.Max(avg.Item2, recentAvg.Item2), + Math.Max(avg.Item3, recentAvg.Item3)); + + _logger.LogDebug("[ChannelRec] {ApName} {Band} ch{Ch}: 7d avg u={U7:F1}% i={I7:F1}% tx={T7:F1}%, " + + "1d avg u={U1:F1}% i={I1:F1}% tx={T1:F1}% ({Count} samples), using max", + ap.Name, band, ch, + dataPoints.Average(d => d.Util), dataPoints.Average(d => d.Interf), dataPoints.Average(d => d.TxRetry), + recentAvg.Item1, recentAvg.Item2, recentAvg.Item3, recentCurrentChannel.Count); + } + + perChannel[ch] = avg; + } + result[band][macLower] = perChannel; } - result[band][macLower] = perChannel; } } - } - context.Stress = result; + context.Stress = result; } catch (Exception ex) { From 5ce1a08f830227114831822f63198425584ad45a Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Sun, 19 Jul 2026 17:09:32 -0500 Subject: [PATCH 03/23] UniFi Device Bridge support: bridged-device names and throughput on the LAN Flow Map (#1027) * Absorb UDB device names and bridged-client throughput Two related gaps for UniFi Device Bridge (UDB) setups (e.g. a Protect camera wirelessly bridged onto the LAN): Names: UDB-bridged clients carry no user name/display_name, only an auto hostname, but expose a friendly name in unifi_device_info_from_ucore (e.g. '[Camera] Driveway'). Add that field to UniFiClientResponse and use it as a label fallback (display name > name > ucore name > hostname > MAC) on the 2D/3D map and Client Performance. Rates: a UDB uplinks over Wi-Fi with the bridged client on its single downlink port, whose own client counters UniFi reports as zero. Its real throughput is on the UDB's own port_table. Fold DeviceBridge into the mesh-AP backhaul synthesis (new contributor reading the bridge's downlink port, gated to DeviceBridge) so the UDB's uplink link shows the flow, and source the bridged client's leaf-link rate from that same aggregate (gated to bridge parents). Switches, their clients, and mesh APs are untouched. * Record UDB bridge rates to InfluxDB for historic map playback The live UDB rate fix routes through LanFabricAggregator + MonitoringLiveStats, which are in-memory only. Historic LAN-flow-map playback re-derives rates from the durable interface_counters measurement instead - which works for switches/ APs/mesh (their bytes are in an SNMP port or vwiresta series) but not for a UDB, whose throughput lives solely on its own port_table (a UniFi-API signal, never SNMP). So the UDB uplink and its bridged-client leaf were blank in playback. Persist each UDB's summed downlink port_table rate to interface_counters under a synthetic 'bridge-downlink' series (BridgeInterfaceRecorder), from the shared recording path so both directly-monitored and agent-relayed sites record it. Teach the historic MeshBackhaul and WiredClient resolvers to read that series (the latter via the existing BridgeParentMac marker), matching the live directions. No new measurement; switches/APs/mesh playback is unchanged. * Use a generic example device name in doc comments * Keep UDB device badge symmetric in historic playback The historic node-badge fabric-sum treated the UDB's synthetic bridge-downlink series as switch fabric, summing its single directional flow into ingress/egress (low in, high out) - so a bridge looked asymmetric during playback while live showed it symmetric. Exclude the bridge-downlink series from the fabric-sum so the UDB falls back to adjacent-link summing, matching the live badge. Real switches never carry that series, so they're unaffected. --- .../Models/UniFiClientResponse.cs | 30 ++++++ src/NetworkOptimizer.UniFi/UniFiDiscovery.cs | 7 ++ .../Services/AgentProbeResultSink.cs | 4 + .../Services/BridgeInterfaceRecorder.cs | 101 ++++++++++++++++++ .../Services/ClientDashboardService.cs | 11 +- .../Services/LanFabricAggregator.cs | 70 +++++++----- .../Services/LanFlowMap/LanFlowMapModels.cs | 7 ++ .../Services/LanFlowMap/LanFlowMapService.cs | 65 ++++++++++- .../Services/MonitoringCollectionAgent.cs | 7 +- 9 files changed, 272 insertions(+), 30 deletions(-) create mode 100644 src/NetworkOptimizer.Web/Services/BridgeInterfaceRecorder.cs diff --git a/src/NetworkOptimizer.UniFi/Models/UniFiClientResponse.cs b/src/NetworkOptimizer.UniFi/Models/UniFiClientResponse.cs index d847c619e..1a40cff9b 100644 --- a/src/NetworkOptimizer.UniFi/Models/UniFiClientResponse.cs +++ b/src/NetworkOptimizer.UniFi/Models/UniFiClientResponse.cs @@ -255,6 +255,36 @@ public class UniFiClientResponse /// [JsonPropertyName("roam_count")] public int? RoamCount { get; set; } + + /// + /// UniFi ecosystem device metadata surfaced by the console (ucore) for an adopted/bridged + /// device that shows up as a client - e.g. a UniFi Protect camera bridged onto the LAN via a + /// UniFi Device Bridge, or a UNAS. Carries the friendly device name and product line/model. + /// Null for ordinary clients. + /// + [JsonPropertyName("unifi_device_info_from_ucore")] + public UniFiUcoreDeviceInfo? UnifiDeviceInfoFromUcore { get; set; } +} + +/// +/// UniFi ecosystem device info (from the console's ucore) for an adopted/bridged device that +/// appears as a client - e.g. a UniFi Protect camera on a UniFi Device Bridge. The friendly +/// is what UniFi shows for the device; the product fields are kept for +/// identifying the device line/model. +/// +public class UniFiUcoreDeviceInfo +{ + /// Friendly device name as configured in the owning app, e.g. "[Camera] Front Door". + [JsonPropertyName("name")] + public string? Name { get; set; } + + /// Product line, e.g. "PROTECT", "NETWORK", "UNAS". + [JsonPropertyName("product_line")] + public string? ProductLine { get; set; } + + /// Short product/model name, e.g. "UVC G6 Pro Bullet". + [JsonPropertyName("product_shortname")] + public string? ProductShortname { get; set; } } /// diff --git a/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs b/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs index d88c032c7..da7bd2ce8 100644 --- a/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs +++ b/src/NetworkOptimizer.UniFi/UniFiDiscovery.cs @@ -381,6 +381,7 @@ public async Task> DiscoverClientsAsync(CancellationToken Hostname = c.Hostname, Name = c.Name, DisplayName = displayNames.TryGetValue(c.Mac.ToLowerInvariant(), out var dn) ? dn : string.Empty, + UcoreName = c.UnifiDeviceInfoFromUcore?.Name ?? string.Empty, IpAddress = ipAddress ?? string.Empty, Network = c.Network, NetworkId = c.NetworkId, @@ -885,6 +886,12 @@ public class DiscoveredClient /// Prefer this over / for display labels. /// public string DisplayName { get; set; } = string.Empty; + /// + /// Friendly name of a bridged/adopted UniFi ecosystem device (from the console's ucore), + /// e.g. a UniFi Protect camera's "[Camera] Front Door" surfaced through a UniFi Device Bridge. + /// Empty for ordinary clients. Used as a display-label fallback below the user-set Name. + /// + public string UcoreName { get; set; } = string.Empty; public string IpAddress { get; set; } = string.Empty; public string Network { get; set; } = string.Empty; public string NetworkId { get; set; } = string.Empty; diff --git a/src/NetworkOptimizer.Web/Services/AgentProbeResultSink.cs b/src/NetworkOptimizer.Web/Services/AgentProbeResultSink.cs index f23ec7c8b..1127fa448 100644 --- a/src/NetworkOptimizer.Web/Services/AgentProbeResultSink.cs +++ b/src/NetworkOptimizer.Web/Services/AgentProbeResultSink.cs @@ -731,6 +731,10 @@ await influx.WriteInterfaceCountersAsync( // fast tier's RecordInterfaceAggregate(mac, out, in) convention. liveStats.RecordInterfaceAggregate(mDev.Mac, m.Out, m.In, aggNow); fabric.WriteAggregates(console.Devices, liveStats, aggNow); + // Persist UDB downlink port_table rates to interface_counters for the historic + // resolver - identical to the directly-monitored fast tier (in-memory-only live path, + // and a UDB has no SNMP interface series to re-derive from during playback). + BridgeInterfaceRecorder.Record(fabric, console.Devices, influx, aggNow); } // Reconcile the InterfaceNameMap (friendly name, negotiated speed, port number, diff --git a/src/NetworkOptimizer.Web/Services/BridgeInterfaceRecorder.cs b/src/NetworkOptimizer.Web/Services/BridgeInterfaceRecorder.cs new file mode 100644 index 000000000..ebe221814 --- /dev/null +++ b/src/NetworkOptimizer.Web/Services/BridgeInterfaceRecorder.cs @@ -0,0 +1,101 @@ +using NetworkOptimizer.Core.Enums; +using NetworkOptimizer.Storage.Models; +using NetworkOptimizer.Storage.Services; +using NetworkOptimizer.UniFi.Models; + +namespace NetworkOptimizer.Web.Services; + +/// +/// Persists each UniFi Device Bridge (UDB) downlink port_table rate to the interface_counters +/// InfluxDB measurement so the LAN Flow Map's HISTORIC resolver can re-derive it. +/// +/// The live path reads a UDB's rate from + MonitoringLiveStats, +/// which are in-memory only. Every other device's boundary aggregate has a durable +/// interface_counters series to re-derive from during playback (an SNMP uplink port, or a mesh +/// AP's vwiresta interface). A UDB's throughput uniquely lives on its own port_table - a +/// UniFi-API signal that is never SNMP-polled - so without this it has no durable series at all. +/// +/// One summed "bridge-downlink" series is written per UDB, matching the live WriteAggregates +/// aggregate. Shared by the directly-monitored fast tier (MonitoringCollectionAgent) and the +/// agent-relayed path (AgentProbeResultSink) so both site types record identically. +/// +public static class BridgeInterfaceRecorder +{ + /// + /// Synthetic interface name for a UDB's summed downlink port_table series in + /// interface_counters. The historic MeshBackhaul and WiredClient resolvers match on it. + /// + public const string DownlinkIfName = "bridge-downlink"; + + /// + /// Writes the downlink port_table rate of every DeviceBridge in + /// to interface_counters. Call right after + /// (its live sibling), once has run so + /// PortRate is populated. No-op until a rate is available (needs a prior byte sample to delta). + /// + public static void Record( + LanFabricAggregator fabric, + IReadOnlyList devices, + MonitoringInfluxClient influx, + DateTime now) + { + if (!influx.IsConfigured) return; + + foreach (var dev in devices) + { + if (dev.DeviceType != DeviceType.DeviceBridge || dev.PortTable == null) continue; + + var devMac = dev.Mac.ToLowerInvariant().Replace("-", ":"); + double downBps = 0, upBps = 0; + long txBytes = 0, rxBytes = 0; + long? speedBps = null; + bool anyRate = false; + + foreach (var port in dev.PortTable) + { + if (port.IsUplink) continue; + var rate = fabric.PortRate(devMac, port.PortIdx); + if (!rate.HasValue) continue; + + // PortRate tuple: DownBps = RX-derived = bytes FROM the bridged client = upload + // (upstream, toward the gateway); UpBps = TX-derived = bytes TO the client = + // download (downstream). Persist in interface_counters' convention, where + // rateIn = downstream/download and rateOut = upstream/upload, so the historic + // resolvers read it with the same (Down = rateIn, Up = rateOut) mapping they use + // for a vwiresta or SNMP series. + downBps += rate.Value.UpBps; + upBps += rate.Value.DownBps; + txBytes += port.TxBytes; + rxBytes += port.RxBytes; + if (port.Speed > 0) speedBps = Math.Max(speedBps ?? 0, (long)port.Speed * 1_000_000L); + anyRate = true; + } + + if (!anyRate) continue; + + _ = influx.WriteInterfaceCountersAsync( + deviceMac: devMac, + ifName: DownlinkIfName, + portId: null, + direction: InterfaceDirection.Unknown, + bytesIn: txBytes, // rateIn = download -> port TX bytes (toward the client) + bytesOut: rxBytes, // rateOut = upload -> port RX bytes (from the client) + rateInBps: downBps, + rateOutBps: upBps, + speedBps: speedBps, + operStatus: 1, + errorsIn: 0, + errorsOut: 0, + discardsIn: 0, + discardsOut: 0, + hcCounters: true, + ucastPktsIn: null, + ucastPktsOut: null, + mcastPktsIn: null, + mcastPktsOut: null, + bcastPktsIn: null, + bcastPktsOut: null, + timestamp: now); + } + } +} diff --git a/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs b/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs index c737a88dd..713b415f4 100644 --- a/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs +++ b/src/NetworkOptimizer.Web/Services/ClientDashboardService.cs @@ -101,6 +101,7 @@ public async Task> GetSelectableClientsAsync() c.BestIp!, displayNames.TryGetValue(c.Mac.ToLowerInvariant(), out var dn) ? dn : !string.IsNullOrWhiteSpace(c.Name) ? c.Name + : c.UnifiDeviceInfoFromUcore?.Name is { Length: > 0 } ucore ? ucore : !string.IsNullOrWhiteSpace(c.Hostname) ? c.Hostname : c.BestIp!, c.IsWired)) .OrderBy(c => c.Name, StringComparer.OrdinalIgnoreCase) @@ -951,14 +952,18 @@ private async Task StoreSignalLogAsync( private ClientIdentity MapClientToIdentity(UniFiClientResponse client, string? displayName = null) { + // Bridged UniFi ecosystem devices (e.g. a Protect camera on a UniFi Device Bridge) have + // no user Name/display_name but expose a friendly ucore name like "[Camera] Front Door". + var ucoreName = client.UnifiDeviceInfoFromUcore?.Name; return new ClientIdentity { Mac = client.Mac, // Prefer UniFi's system-selected display name (v2 active-clients, e.g. a - // fingerprint-derived "Apple TV") over the raw stat/sta name, matching Client - // Stats and the offline-identity path so an unnamed device never shows as a MAC. + // fingerprint-derived "Apple TV") over the raw stat/sta name, then the ucore device + // name, matching Client Stats/the map so an unnamed device never shows as a MAC. Name = !string.IsNullOrEmpty(displayName) ? displayName - : !string.IsNullOrEmpty(client.Name) ? client.Name : null, + : !string.IsNullOrEmpty(client.Name) ? client.Name + : !string.IsNullOrEmpty(ucoreName) ? ucoreName : null, Hostname = !string.IsNullOrEmpty(client.Hostname) ? client.Hostname : null, Ip = client.Ip, IsWired = client.IsWired, diff --git a/src/NetworkOptimizer.Web/Services/LanFabricAggregator.cs b/src/NetworkOptimizer.Web/Services/LanFabricAggregator.cs index 3b1c047d3..6756a510a 100644 --- a/src/NetworkOptimizer.Web/Services/LanFabricAggregator.cs +++ b/src/NetworkOptimizer.Web/Services/LanFabricAggregator.cs @@ -224,30 +224,29 @@ public void WriteAggregates(IReadOnlyList devices, Monitori // would produce a one-burst / many-zeroes pattern here). } - // Second pass: mesh-uplinked APs need a custom aggregate because - // UniFi's device-level stat.tx_bytes / rx_bytes doesn't reliably - // include traffic shuttled across the wireless backhaul - the - // AP-fallback above can read low or zero even when the AP is - // relaying a lot of traffic for downstream gear and its own - // wireless clients. Wired APs don't need this: their parent - // switch port already sees every byte (wireless clients included) - // because that traffic exits the AP via Ethernet. Mesh APs have - // no such port to read, so we synthesize the aggregate from two - // contributors: - // (a) downstream UniFi devices (switch or another AP plugged - // into the mesh AP's Ethernet downlink) - their boundary - // aggregates were just written in the first pass. - // (b) wireless clients directly associated to this mesh AP - - // their TX/RX throughput maps onto the backhaul flow. - // NetworkPathAnalyzer treats device.Uplink.Type == "wireless" as - // the mesh marker; mirror that here for consistency with how the - // speed-test path tracer identifies mesh hops. - foreach (var meshAp in devices.Where(d => - d.DeviceType == DeviceType.AccessPoint + // Second pass: wirelessly-uplinked devices - mesh APs AND single-port bridges + // (UDBs) - need a custom aggregate because UniFi's device-level stat.tx_bytes / + // rx_bytes doesn't reliably include traffic shuttled across the wireless backhaul, + // and there's no parent switch port to read (the first pass found nothing for them). + // Wired APs/switches don't need this: their parent port already sees every byte. + // We synthesize the aggregate from whichever of these contributors apply: + // (a) downstream UniFi devices (switch or another AP plugged into an Ethernet + // downlink) - their boundary aggregates were just written in the first pass. + // (b) wireless clients directly associated to a mesh AP - their TX/RX throughput + // maps onto the backhaul flow. + // (c) for a UDB, the bridged client is a plain wired client (neither a UniFi child + // nor a wifi client), so its throughput lives on the bridge's OWN downlink + // port_table; read that. Restricted to DeviceBridge so a mesh AP never + // double-counts its downstream, which is already covered by (a). + // NetworkPathAnalyzer treats device.Uplink.Type == "wireless" as the mesh marker; + // mirror that here for consistency with how the speed-test path tracer identifies + // mesh hops. + foreach (var dev in devices.Where(d => + (d.DeviceType == DeviceType.AccessPoint || d.DeviceType == DeviceType.DeviceBridge) && d.Uplink != null && string.Equals(d.Uplink.Type, "wireless", StringComparison.OrdinalIgnoreCase))) { - var meshMac = NormalizeMac(meshAp.Mac); + var devMac = NormalizeMac(dev.Mac); double sumIn = 0, sumOut = 0; bool anyContribution = false; @@ -255,7 +254,7 @@ public void WriteAggregates(IReadOnlyList devices, Monitori foreach (var child in devices) { if (child.Uplink == null || string.IsNullOrEmpty(child.Uplink.UplinkMac)) continue; - if (!string.Equals(NormalizeMac(child.Uplink.UplinkMac), meshMac, StringComparison.OrdinalIgnoreCase)) continue; + if (!string.Equals(NormalizeMac(child.Uplink.UplinkMac), devMac, StringComparison.OrdinalIgnoreCase)) continue; var stats = liveStats.GetForDevice(child.Mac); if (stats == null || !stats.LastRateUpdate.HasValue) continue; sumIn += stats.RateInBps ?? 0; @@ -267,7 +266,8 @@ public void WriteAggregates(IReadOnlyList devices, Monitori // is AP -> client (downloads, gateway-relative), Rx is the // reverse (uploads). Sum onto the same RateIn / RateOut sides // the children write so the totals stay direction-consistent. - foreach (var wc in liveStats.GetWifiClientsForAp(meshAp.Mac)) + // A UDB is not an AP, so this yields nothing for it. + foreach (var wc in liveStats.GetWifiClientsForAp(dev.Mac)) { var rx = wc.RxThroughputBps ?? 0; var tx = wc.TxThroughputBps ?? 0; @@ -279,17 +279,37 @@ public void WriteAggregates(IReadOnlyList devices, Monitori } } + // (c) UDB bridged wired client: its bytes live on the bridge's own downlink + // port_table. Same direction convention as a parent reading a port toward a + // child (port RX = uploads from the client, TX = downloads to it), so add + // (DownBps, UpBps) onto the same (sumIn, sumOut) sides as (a)/(b). DeviceBridge + // only - a mesh AP's own downstream is already counted in (a). + if (dev.DeviceType == DeviceType.DeviceBridge && dev.PortTable != null) + { + foreach (var port in dev.PortTable) + { + if (port.IsUplink) continue; + var portRate = PortRate(devMac, port.PortIdx); + if (portRate.HasValue) + { + sumIn += portRate.Value.DownBps; + sumOut += portRate.Value.UpBps; + anyContribution = true; + } + } + } + // SNMP first-pass (vwiresta) is the most accurate source. Only // overwrite if SNMP didn't get a chance (e.g. AP unreachable via // SNMP) - i.e. the live-stats entry has no aggregate yet. if (anyContribution) { - var existing = liveStats.GetForDevice(meshAp.Mac); + var existing = liveStats.GetForDevice(dev.Mac); bool snmpAlreadySet = existing?.RateInBps.HasValue == true || existing?.RateOutBps.HasValue == true; if (!snmpAlreadySet) { - liveStats.RecordInterfaceAggregate(meshAp.Mac, sumIn, sumOut, now); + liveStats.RecordInterfaceAggregate(dev.Mac, sumIn, sumOut, now); } } } diff --git a/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapModels.cs b/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapModels.cs index 46fbfeacf..1c7859d33 100644 --- a/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapModels.cs +++ b/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapModels.cs @@ -150,6 +150,13 @@ public class LanLink /// Stable correlation key for wired throughput chain: device MAC + ifName (spec 3.7). /// Also the lookup key for MonitoringLiveStats.GetPortRate per-port live tick refreshes. public string? PortKey { get; set; } + + /// Server-side only. Set to the parent's device MAC ONLY for a wired-client link whose + /// parent is a single-port UniFi Device Bridge (UDB). The live-rate pass sources this leaf's + /// rate from the bridge's device aggregate, because the bridged client's own wired byte + /// counters are always zero. Null for every other link, so no existing case is affected. + [System.Text.Json.Serialization.JsonIgnore] + public string? BridgeParentMac { get; set; } } public enum LanCloudKind diff --git a/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs b/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs index 72a41f401..859ef5072 100644 --- a/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs +++ b/src/NetworkOptimizer.Web/Services/LanFlowMap/LanFlowMapService.cs @@ -352,6 +352,24 @@ public async Task GetLiveUpdateAsync(CancellationToken ct } } } + // UDB (single-port bridge) leaf: the bridged client's own wired counters are + // always zero, so source the rate from the bridge's device aggregate (the same + // value shown on the UDB's wireless-uplink link, since it's a single flow). + // BridgeParentMac is set only for DeviceBridge parents, so switch/AP clients + // never take this path and keep their existing client-counter behavior. + if (rates == null && !string.IsNullOrEmpty(link.BridgeParentMac)) + { + var bridge = _liveStats.GetForDevice(link.BridgeParentMac); + if (bridge != null && bridge.LastRateUpdate.HasValue) + { + rates = new LinkLiveRates + { + DownstreamBps = bridge.RateOutBps ?? 0, + UpstreamBps = bridge.RateInBps ?? 0, + AsOf = bridge.LastRateUpdate.Value, + }; + } + } // Fallback: UniFi client stats (for switches without SNMP). // TX from the client's perspective = upload = upstream on the link. if (rates == null) @@ -660,6 +678,14 @@ public async Task GetHistoricUpdateAsync(DateTime at, && !p.IfName.Contains('.')) .OrderBy(p => Math.Abs((p.Time - at).TotalMilliseconds)) .FirstOrDefault(); + // UDB single-port bridge: no vwiresta interface. Its downlink + // port_table rate is persisted under a synthetic "bridge-downlink" + // series (BridgeInterfaceRecorder), stored in the same rateIn = + // downstream convention, so it maps through the block below. + resolved ??= cPts + .Where(p => string.Equals(p.IfName, BridgeInterfaceRecorder.DownlinkIfName, StringComparison.OrdinalIgnoreCase)) + .OrderBy(p => Math.Abs((p.Time - at).TotalMilliseconds)) + .FirstOrDefault(); } else { @@ -729,6 +755,25 @@ public async Task GetHistoricUpdateAsync(DateTime at, rates = MapPortToLinkRates(link, closest.RateInBps ?? 0, closest.RateOutBps ?? 0, closest.Time); } } + // UDB bridged leaf: the client's own wired counters are always zero, so source + // the rate from the bridge's persisted downlink series (mirrors the live path's + // BridgeParentMac substitution). Set only for DeviceBridge parents, so switch/AP + // clients keep the wired_client fallback below untouched. + if (rates == null && !string.IsNullOrEmpty(link.BridgeParentMac) + && ratesByDevice.TryGetValue(link.BridgeParentMac, out var bpts)) + { + var closest = bpts + .Where(p => string.Equals(p.IfName, BridgeInterfaceRecorder.DownlinkIfName, StringComparison.OrdinalIgnoreCase)) + .OrderBy(p => Math.Abs((p.Time - at).TotalMilliseconds)) + .FirstOrDefault(); + if (closest != null) + rates = new LinkLiveRates + { + DownstreamBps = closest.RateInBps ?? 0, + UpstreamBps = closest.RateOutBps ?? 0, + AsOf = closest.Time, + }; + } // Fallback: wired_client from batch pre-fetch if (rates == null) { @@ -788,7 +833,12 @@ public async Task GetHistoricUpdateAsync(DateTime at, var isGw = node.Kind == LanNodeKind.Gateway; var filtered = isGw ? rates.Where(p => System.Text.RegularExpressions.Regex.IsMatch(p.IfName, @"^eth\d+$")) - : rates; + // Exclude the synthetic UDB bridge-downlink series: it's a single directional + // flow, not a switch fabric, so summing it as ingress/egress makes a bridge + // look asymmetric. Dropping it leaves the bridge with no fabric badge, so it + // falls back to adjacent-link summing (symmetric) - matching the live badge, + // which has no fabric series for a UDB either. + : rates.Where(p => !string.Equals(p.IfName, BridgeInterfaceRecorder.DownlinkIfName, StringComparison.OrdinalIgnoreCase)); var closestRates = filtered .GroupBy(p => p.Time) .OrderBy(g => Math.Abs((g.Key - at).TotalMilliseconds)) @@ -1382,6 +1432,13 @@ private void BuildClientLeaves( if (string.IsNullOrEmpty(node.SwitchPortName) && !string.IsNullOrEmpty(port.Name)) node.SwitchPortName = port.Name; } + + // A UDB (single-port bridge) never reports non-zero wired byte counters for the + // client behind it, so tag this leaf to source its live rate from the bridge's + // device aggregate instead. Only DeviceBridge parents are tagged - switch/AP + // clients keep their existing (working) client-counter path untouched. + if (parentDev.DeviceType == DeviceType.DeviceBridge) + link.BridgeParentMac = parentMac; } } else if (!c.IsWired) @@ -2035,6 +2092,9 @@ private static string ResolveClientLabel(NetworkOptimizer.UniFi.DiscoveredClient { if (!string.IsNullOrWhiteSpace(c.DisplayName)) return c.DisplayName; if (!string.IsNullOrWhiteSpace(c.Name)) return c.Name; + // Bridged UniFi ecosystem devices (Protect cameras, UNAS) carry no user Name/DisplayName + // but expose a friendly ucore name; prefer it over the auto-generated hostname. + if (!string.IsNullOrWhiteSpace(c.UcoreName)) return c.UcoreName; if (!string.IsNullOrWhiteSpace(c.Hostname)) return c.Hostname; return string.IsNullOrEmpty(c.Mac) ? "unknown" : c.Mac; } @@ -2077,6 +2137,9 @@ private async Task FetchHistoricDataAsync( var (mac, _) = ParsePortKey(link.PortKey); if (!string.IsNullOrEmpty(mac)) deviceMacs.Add(mac); } + // A UDB-bridged client leaf sources its historic rate from the bridge's persisted + // downlink series, so make sure the bridge's interface rows are fetched. + if (!string.IsNullOrEmpty(link.BridgeParentMac)) deviceMacs.Add(link.BridgeParentMac); } var ratesByDevice = new Dictionary>( diff --git a/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs b/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs index 9dbc47d78..8bbb25756 100644 --- a/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs +++ b/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs @@ -512,7 +512,12 @@ private async Task FastTierCollectAsync(MonitoringSettings settings, Cancellatio // synthesis, and gateway WAN rates, with all the direction conventions and // fallbacks. Shared verbatim with the agent-relayed path (AgentProbeResultSink) // via LanFabricAggregator so secondary sites compute identical numbers. - _fabric.WriteAggregates(devices, _liveStats, DateTime.UtcNow); + var aggNow = DateTime.UtcNow; + _fabric.WriteAggregates(devices, _liveStats, aggNow); + // Persist UDB downlink port_table rates to interface_counters so the historic LAN-flow-map + // resolver can re-derive them (the live path above is in-memory only, and a UDB has no + // SNMP interface series). Shared verbatim with the agent-relayed path. + BridgeInterfaceRecorder.Record(_fabric, devices, _influx, aggNow); } /// From 9df7a3023ee7ddaf2e3c953891d3e22e71fbeaa5 Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:59:22 -0500 Subject: [PATCH 04/23] ISP Health: involvement-weighted Transit Health, destination-absolved transit jitter, off-path hop guidance (#1028) * Upstream discovery: always attach the Level 3 transit witness when near-transit Previously the anycast witness (4.2.2.2) was injected only as a fallback, when no real AS3356 router cleared the reachability gate. When real Level 3 hops respond, they'd suppress it - so a site with pingable Level 3 hops never got the stable anycast anchor, and ISP Health's Lumen transit signal drifted POP-to-POP across discovery runs. Drop that suppression: whenever a curated transit ASN is genuinely near-transit (and its witness itself clears the gate), attach the witness alongside any real routers. Real hops are kept, never replaced - the anycast endpoint is usually a more distant POP than the closest real hop. Its samples fold into the ASN's transit tier in ISP Health automatically. Same-address dedup on write means a hand-added witness at the same address is respected, not duplicated. Generalizes to every entry in the witness table. * Upstream discovery: traceroute the transit witness so it carries ancestry The witness was injected as a ping-only target, so it landed in UpstreamDiscoveries with empty AncestorHopIps and could never satisfy FarClusterRoutesThroughNear - meaning ISP Health could never use the witness's clean end-to-end jitter to absolve a jittery near hop it routes through, even though the whole absolution mechanism already exists for traced destinations. Trace the witness at injection and append the result to the discovery traces. PersistHopOrderAsync then records the real transit hops preceding it as its ancestry, unchanged. The proof is honest: ancestry comes from the anycast endpoint's actual path, so an absolve only fires when that path genuinely traverses the monitored hop; a different POP yields no overlap and no absolve. No schema change (AncestorHopIps already exists); scorer-side absolution and the ancestor gate are already covered by IspHealthScorerTests. * ISP Health: destinations absolve transit jitter + involvement-weighted Transit Health Arm A: transit ASNs are now absolved by any monitored destination (or another transit ASN) proven to route through them, routes-through gated on stored ancestry - the same clean-end-to-end upper-bound logic already used for ISP hops. Folded with the existing within-ASN far-cluster absolution; strict, never on faith. Arm 4: Transit Health is now a weighted average - each transit ASN weighted by its internet-host involvement (share of monitored destinations proven to route through it): floor 0.25 + 0.75 * reach/maxReach, so a graded ASN never zeroes but a side-path transit counts as little as a quarter of the main-path one. When no host is attributable (transit invisible on destination traces), all weights collapse to equal - identical to the prior plain average. A fraction-icon tooltip surfaces the share/weight, shown only when involvement differentiates. No schema change - both compute from ancestry already loaded in the scorer. Tests cover destination->transit absolution and involvement weighting. * ISP Health: floor transit involvement at 25% on peered sites (true zero) The involvement weighting conflated two different zeros: "can't attribute" (no hop order / no destinations) and "attributed, and the answer is zero" (a peered site whose destinations cross no transit). The first should fall back to equal weight; the second is a true-zero involvement and should sit at the 25% floor. Gate the involvement weighting on attribution being KNOWN (hop order + at least one destination) rather than on maxReach > 0. When known but no transit carries a host, every transit ASN floors at 25% and shows the fraction-icon tooltip ("0 of N hosts") instead of silently reverting to equal weight with no icon. The dimension score is unchanged where the flooring is uniform; the point is correct, visible attribution. * ISP Health: blend uninvolved transit toward neutral, pie-slice involvement icon Scoring: a transit ASN now contributes weight*own + (1-weight)*100 to Transit Health, involvement-weighted - so a transit carrying none of your hosts can't drag the number (its uninvolved share reads neutral), while the 25% floor keeps a graded ASN from vanishing. Fixes the case where a peered site's unused transit (e.g. Cogent at 74) pulled Transit Health down via the plain average, even though it carries zero monitored hosts. Involvement now lives on IspAsnHealth as the single source for both the dimension and the card. UI: replaced the wrapping text glyph with a pie-slice SVG that fills to the involvement weight, kept on the same line as the ASN name (the factor label is a flex column, so name+icon are now grouped in a row), and added it to the Networks on Your Path cards too. data-tooltip carries the host fraction and weight. * ISP Health: transit involvement tooltip explains the 25% floor (asymmetric routing) The floored-transit tooltip now makes clear the ASN is still scored, not dropped: our involvement is measured on the forward path (traceroute), but a transit we never cross outbound often carries the return path from popular services (hot-potato / asymmetric routing) - so it's held at the 25% minimum rather than zeroed. The carrying-hosts tooltip is clarified to "on the forward path". * ISP Health: trim transit involvement tooltip to one line * ISP Health: transit tooltip says 'internet targets', not 'hosts' * ISP Health: flag off-path jittery ISP hops with a disable hint linking to Latency targets An ISP-network hop that answers pings but appears on no discovery trace (HopNumber 0 - e.g. an OLT/CMTS that ICMP-deprioritizes traceroute) isn't a forward-path jitter signal, yet its jitter still drags ISP Network. When such a hop is off the traced path AND its jitter score is a real deficit (<=70), and it isn't the graded nearest hop, surface a "Not on path" hint on its ISP Health row with a tooltip; clicking it switches to the Setup tab and highlights the target in Latency targets to pause it. Detection reuses the existing per-target hop distance (HopNumber 0 = never traced); no new discovery data. IspHealthPanel raises OnReviewTarget; Monitoring maps the TargetId to the row and jumps once the card mounts. * ISP Health: disable hint jumps to the right tab; commonize + align the hop badges - Fix: the "Not on path" hint switched to the Setup tab, but Latency targets lives on the Network Performance tab (same as JumpToLatencyTargetAsync). Jump there so the target actually highlights. - CSS: extract a shared .isp-target-badge base for the "Lowest RTT" and "Not on path" badges (margin, padding, radius, font, uppercase, spacing, vertical-align); only color/cursor/hover stay per-badge. Bumps margin to 0.5rem and vertically centers them against the hop name. * TODO: AWS S3/DynamoDB endpoints for transit trace exposure (Monitoring) * TODO: DynamoDB regional endpoints (ICMP-pingable) for transit trace exposure --- TODO.md | 17 ++ .../Components/Pages/Monitoring.razor | 23 ++- .../Shared/Monitoring/IspHealthPanel.razor | 73 ++++++-- .../Monitoring/IspHealth/IspHealthModels.cs | 51 ++++++ .../Monitoring/IspHealth/IspHealthScorer.cs | 158 +++++++++++++++++- .../Monitoring/IspHealth/IspHealthService.cs | 5 + .../Monitoring/UpstreamTracerService.cs | 46 +++-- src/NetworkOptimizer.Web/wwwroot/css/app.css | 46 ++++- .../IspHealth/IspHealthScorerTests.cs | 157 ++++++++++++++++- 9 files changed, 537 insertions(+), 39 deletions(-) diff --git a/TODO.md b/TODO.md index f316e4f56..4825dad03 100644 --- a/TODO.md +++ b/TODO.md @@ -321,6 +321,23 @@ The following were implemented in the WiFi Optimizer feature: ## Monitoring +### Upstream path discovery: DynamoDB regional endpoints for transit trace exposure (Setup tab) +Transit Health involvement weighting and destination->transit jitter absolution both key off which +monitored internet targets a transit ASN provably carries (trace ancestry). The current CDN/DNS +targets peer at the local IX, so they cross no transit and give the attribution nothing to work with. +AWS DynamoDB regional endpoints ARE ICMP-pingable and DO ride paid transit, so they're good extra +trace targets (and can double as monitored targets): + + dynamodb..amazonaws.com (nearest region(s) first, e.g. us-west-2, us-west-1, us-east-2, us-east-1) + +- **Not anycast** - resolve + latency-rank to the nearest region(s), else you trace a transcontinental + path that misrepresents the transit. +- **Attribution still needs work (observed in testing):** the endpoint pings and the path is NOT + peered, but the intermediate transit hops don't answer traceroute (all stars) - so the current + ancestry can't tell WHICH transit ASN it crossed. DynamoDB gives a clean AWS target that genuinely + transits; surfacing which transit still needs the responding-hop-ASN capture (record every + responding hop's ASN, not just monitored-target IPs), and pure-star segments recover nothing. + ### Investigation Functions (Network Performance tab) The Investigate card currently jumps the latency charts to the most recent **packet-loss** and **loaded-loss** events and steps event-to-event (coalesced, peak-loss minute). Ideas to extend it: diff --git a/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor b/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor index 06ad8a279..656627eea 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor @@ -369,7 +369,7 @@ else { @FlakyTargetsBanner } - + } @* ──────────────── Tab: Network Performance ──────────────── *@ @@ -2828,6 +2828,19 @@ else await _latencyTargetsCard.ExpandAndHighlightAsync(targetId); } + // ISP Health disable hint (off-path, jittery hop): switch to the Network Performance tab (where + // Latency targets lives, same as JumpToLatencyTargetAsync), then highlight the target once its + // card mounts (handled in OnAfterRenderAsync via _pendingLatencyJumpId). + private int? _pendingLatencyJumpId; + + private void ReviewIspHopInLatencyTargets(string targetId) + { + var id = _targets.FirstOrDefault(t => t.TargetId == targetId)?.Id; + if (id == null) return; + _pendingLatencyJumpId = id; + SetActiveTab("performance"); + } + private async Task DismissLanFlakyHintAsync(MonitoringTarget target) { try @@ -3516,6 +3529,14 @@ else protected override async Task OnAfterRenderAsync(bool firstRender) { + // Pending jump from the ISP Health disable hint: once the Network Performance tab's Latency + // targets card has mounted, expand + highlight the target, then clear so it fires once. + if (_pendingLatencyJumpId is int jumpId && _latencyTargetsCard != null) + { + _pendingLatencyJumpId = null; + await _latencyTargetsCard.ExpandAndHighlightAsync(jumpId); + } + // The chart containers only exist while the console is connected - the whole // tab region renders behind the connection banner. Without this guard, a page // opened during a console outage (agent-backed site waiting for its tunnel, diff --git a/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor b/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor index 8045df428..7a362a819 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor @@ -1,5 +1,6 @@ @using NetworkOptimizer.Web.Services.Monitoring.IspHealth @using NetworkOptimizer.Storage.Models +@using System.Globalization @implements IDisposable @inject IspHealthService IspHealth @inject NavigationManager NavigationManager @@ -344,7 +345,13 @@ else @target.Name @if (target.IsGradedHop) { - Lowest RTT + Lowest RTT + } + @if (target.SuggestDisable) + { + }
@@ -420,15 +427,21 @@ else ? PhysicalLinkInvestigateTooltip(r.PhysicalLinkSelectedKey, r.PhysicalLinkMedium) : "Jump to the most recent matching loss event on the Network Performance charts"; } - @if (investigateUrl != null) - { - @factor.Name - } - else - { - @factor.Name - } +
+ @if (investigateUrl != null) + { + @factor.Name + } + else + { + @factor.Name + } + @if (!string.IsNullOrEmpty(factor.InvolvementTooltip)) + { + @InvolvementPie(factor.Weight, factor.InvolvementTooltip!) + } +
@if (!string.IsNullOrEmpty(factor.ValueText)) { @factor.ValueText @@ -481,7 +494,13 @@ else
-
@(string.IsNullOrEmpty(asn.AsnName) ? $"AS{asn.AsnNumber}" : asn.AsnName)
+
+
@(string.IsNullOrEmpty(asn.AsnName) ? $"AS{asn.AsnNumber}" : asn.AsnName)
+ @if (!string.IsNullOrEmpty(asn.InvolvementTooltip)) + { + @InvolvementPie(asn.InvolvementWeight ?? 1.0, asn.InvolvementTooltip!) + } +
@(isIspAsn ? "ISP network" : "Transit") @(asn.AsnNumber > 0 ? $"· AS{asn.AsnNumber}" : "")
@(asn.OverallScore?.ToString() ?? "--") @@ -650,6 +669,38 @@ else @code { private const int NetworkPreviewCount = 5; + // Raised when the user clicks the disable hint on an off-path, jittery ISP hop; the parent jumps + // to the Latency targets card so they can pause it. Argument is the hop's TargetId. + [Parameter] public EventCallback OnReviewTarget { get; set; } + + // Arm 4: pie-slice icon showing a transit ASN's internet-host involvement weight (0.25-1.0). + private RenderFragment InvolvementPie(double weight, string tooltip) => @ + + ; + + // SVG path for a pie wedge filled from 12 o'clock clockwise by `fraction` of a full turn, on a + // 16x16 viewBox, radius 6. Empty for fraction <= 0; the full-circle case is handled by the caller. + private static string PieSlicePath(double fraction) + { + fraction = Math.Clamp(fraction, 0, 1); + if (fraction <= 0) return string.Empty; + var theta = 2 * Math.PI * fraction; + var ex = 8 + 6 * Math.Sin(theta); + var ey = 8 - 6 * Math.Cos(theta); + var large = fraction > 0.5 ? 1 : 0; + return string.Create(CultureInfo.InvariantCulture, $"M8,8 L8,2 A6,6 0 {large} 1 {ex:0.##},{ey:0.##} Z"); + } + private IspHealthReport? _report; private bool _loading = true; private bool _chartMounted; diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs index d003072ca..9b500049f 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs @@ -53,6 +53,14 @@ public class IspScoreFactor /// What was measured and against which expectation. public string? Description { get; init; } + + /// + /// For transit ASNs (Arm 4): the involvement-weight tooltip, e.g. "Carries 6 of 8 monitored + /// internet hosts - weighted at 100% in Transit Health". Null when involvement doesn't + /// differentiate (no host attributable to any transit, so all fall to equal weight) - the UI + /// shows the fraction icon only when this is set. + /// + public string? InvolvementTooltip { get; init; } } /// One of the three top-level score dimensions. @@ -103,6 +111,31 @@ public class IspAsnHealth /// Median RTT beyond the first clean ISP hop. Null for ISP ASNs. public double? ReachDeltaMs { get; init; } + /// + /// Arm 4 - internet-host involvement, for transit ASNs. Set only when attribution is known + /// (hop order + destinations); left false/null for ISP ASNs and unattributable installs. + /// + public bool ShowInvolvement { get; set; } + + /// How many monitored internet destinations are proven to route through this ASN. + public int InvolvementReach { get; set; } + + /// Total monitored internet destinations (the denominator of the involvement fraction). + public int InvolvementHostTotal { get; set; } + + /// + /// The 0.25-1.0 weight this ASN's own score carries in Transit Health; the remaining (1 - weight) + /// blends to a neutral 100, so a transit carrying none of your hosts can't drag the dimension. + /// Null when involvement isn't shown. + /// + public double? InvolvementWeight { get; set; } + + /// Fraction-icon tooltip built from the involvement fields; null when not shown. + public string? InvolvementTooltip => !ShowInvolvement || InvolvementWeight is not double w ? null + : InvolvementReach > 0 + ? $"Carries {InvolvementReach} of {InvolvementHostTotal} internet targets (forward path), {w * 100:0}% weight" + : $"Off the forward path; held at {w * 100:0}% (likely the return path from popular services)"; + public int? LatencyStabilityScore { get; init; } public int? JitterScore { get; init; } public int? LossScore { get; init; } @@ -143,6 +176,18 @@ public class IspTargetHealth /// True for the first clean hop, the target the access layer idle latency comes from. public bool IsGradedHop { get; init; } + + /// + /// True when this hop answers pings but appears in no discovery trace (HopNumber 0) - e.g. an + /// OLT/CMTS that ICMP-deprioritizes traceroute. Its jitter isn't a forward-path signal. + /// + public bool NotOnTracedPath { get; init; } + + /// + /// True when this hop is off the traced path AND its jitter is materially degrading its score - + /// a strong "you probably want to stop scoring this" signal that drives the ISP Health disable hint. + /// + public bool SuggestDisable { get; init; } } /// An actionable finding or recommendation surfaced on the ISP Health tab. @@ -661,6 +706,12 @@ public class IspHealthInputs ///
public bool HopOrderKnown { get; init; } + /// + /// TargetIds of monitored hops that answer pings but appear in no discovery trace (HopNumber 0) - + /// used to flag likely-disable ISP hops whose jitter isn't a forward-path signal. + /// + public IReadOnlySet NotTracedTargetIds { get; init; } = new HashSet(); + /// /// Window-aggregated physical-link metrics for the one source matched to the WAN, or null /// when no source matched (or the match was ambiguous and unresolved). Drives the Access diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs index 6b8de82ba..a869c5588 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs @@ -91,7 +91,39 @@ public IspHealthReport Score(IspHealthInputs inputs, AccessProfile profile) var accessMedianRtt = SeriesStats.Median( inputs.FirstHopSeries.Where(s => s.RttAvgMs.HasValue).Select(s => s.RttAvgMs!.Value).ToList()); - var transitAsns = inputs.TransitAsnSeries.Select(s => GradeAsn(s, inputs.CongestionEvents, jitterFloor, accessMedianRtt, inputs.InternetMedianDeltaMs)).ToList(); + var transitAsns = GradeTransitAsns(inputs.TransitAsnSeries, inputs.DestinationSeries, inputs.HopOrderKnown, + inputs.CongestionEvents, jitterFloor, accessMedianRtt, inputs.InternetMedianDeltaMs); + + // Arm 4: each transit ASN's "internet host involvement" - how many monitored internet + // destinations are proven to route through it (routes-through gated on stored ancestry). + // Feeds the involvement-weighted Transit dimension below. Zero for every ASN when transit + // is traceroute-invisible on the destination paths, in which case the dimension falls back + // to an equal-weighted average (the prior behavior). + var transitHopIpsByAsn = inputs.TransitAsnSeries + .GroupBy(s => s.AsnNumber) + .ToDictionary(g => g.Key, g => g.SelectMany(s => s.HopIps).Distinct(StringComparer.OrdinalIgnoreCase).ToList()); + var transitReachByAsn = transitHopIpsByAsn.ToDictionary( + kv => kv.Key, + kv => inputs.HopOrderKnown ? inputs.DestinationSeries.Count(d => RoutesThrough(d.AncestorIps, kv.Value)) : 0); + var transitMaxReach = transitReachByAsn.Count > 0 ? transitReachByAsn.Values.Max() : 0; + + // Attribution is "known" when we have the ancestry to prove routes-through AND destinations to + // test against. Then reach == 0 is a TRUE zero (a peered site whose destinations cross no + // transit) and floors the ASN at 25% - distinct from "no ancestry at all", where we can't tell + // and leave involvement unset (equal weight). Attach the involvement to each transit ASN so + // both Transit Health and the Networks on Your Path card read from one source. + var transitAttributionKnown = inputs.HopOrderKnown && inputs.DestinationSeries.Count > 0; + foreach (var a in transitAsns) + { + a.ShowInvolvement = transitAttributionKnown; + a.InvolvementHostTotal = inputs.DestinationSeries.Count; + a.InvolvementReach = transitReachByAsn.TryGetValue(a.AsnNumber, out var rc) ? rc : 0; + a.InvolvementWeight = transitAttributionKnown + ? (transitMaxReach > 0 + ? TransitInvolvementFloor + (1 - TransitInvolvementFloor) * ((double)a.InvolvementReach / transitMaxReach) + : TransitInvolvementFloor) + : null; + } // Every ISP hop is graded; the dimension averages them all. Each hop's jitter is // absolved per-hop and routes-through-gated (a transit ASN or deeper ISP hop only @@ -197,7 +229,7 @@ double Severity(OutageEvent o) IspAsnDimension = ispAsnDimension, TransitAsns = transitAsns, IspAsns = ispAsns, - IspTargets = inputs.IspTargetSeries.Select(s => BuildIspTargetHealth(s, inputs.FirstHopTargetId, ispHopGrades, _options.RttWinsorPercentile)).ToList(), + IspTargets = inputs.IspTargetSeries.Select(s => BuildIspTargetHealth(s, inputs.FirstHopTargetId, ispHopGrades, _options.RttWinsorPercentile, inputs.NotTracedTargetIds)).ToList(), CongestionEvents = inputs.CongestionEvents, PathShifts = inputs.PathShifts, Outages = inputs.Outages, @@ -1000,6 +1032,70 @@ private double ScoreJitterVsFloor(double jitterMs, double? floorMs) /// ICMP-deprioritized hop whose forwarded traffic actually reaches the destination cleanly. /// Hops are also scored against the intra-ASN reach floor (distance, not a fault). /// + /// + /// Grades every transit ASN. Beyond the within-ASN far-cluster absolution baked into each + /// series' , a transit ASN's jitter is additionally + /// absolved (Arm A) by any monitored destination - or any OTHER transit ASN - proven to route + /// through it (routes-through gated on stored ancestry): a clean end-to-end path across the ASN + /// is an upper bound on the ASN's true jitter, so an ICMP-deprioritized transit router isn't + /// penalized for control-plane noise its forwarded traffic never sees. Strict: with no ancestry + /// (hopOrderKnown false) nothing absolves, and a witness only counts where it provably routes + /// through the ASN - never on faith. Falls back to the base within-ASN grade when no witness + /// applies. + /// + private List GradeTransitAsns( + List transitSeries, + List destinationSeries, + bool hopOrderKnown, + List congestionEvents, + double? jitterFloorMs, + double? accessBaselineRtt, + double? internetMedianDeltaMs) + { + // Base grade (within-ASN far-cluster absolution only) - the prior behavior. Serves as the + // fallback and as each ASN's own jitter when it witnesses another ASN. + var baseGrades = transitSeries + .Select(s => (Series: s, Grade: GradeAsn(s, congestionEvents, jitterFloorMs, accessBaselineRtt, internetMedianDeltaMs))) + .ToList(); + + // Destination end-to-end jitter + each transit ASN's base jitter, keyed by the ancestor IPs + // that prove what they route through. Destinations only when ancestry exists (strict). + var destWitnesses = hopOrderKnown + ? destinationSeries + .Select(d => (d.AncestorIps, Jitter: ScoringJitterOf(d.Samples))) + .Where(w => w.Jitter.HasValue) + .Select(w => (w.AncestorIps, Jitter: w.Jitter!.Value)) + .ToList() + : new List<(List AncestorIps, double Jitter)>(); + var transitWitnesses = baseGrades + .Where(b => b.Grade.P95JitterMs.HasValue) + .Select(b => (b.Series.AsnNumber, b.Series.AncestorIps, Jitter: b.Grade.P95JitterMs!.Value)) + .ToList(); + + var grades = new List(); + foreach (var (series, baseGrade) in baseGrades) + { + // The jitter the base grade scored on: near vs this ASN's own farther cluster. + var within = EffectiveLower(series.Samples, series.JitterSourceSamples, ScoringJitterOf); + var witnesses = destWitnesses + .Where(w => hopOrderKnown && RoutesThrough(w.AncestorIps, series.HopIps)) + .Select(w => w.Jitter) + .Concat(transitWitnesses + .Where(w => hopOrderKnown && w.AsnNumber != series.AsnNumber && RoutesThrough(w.AncestorIps, series.HopIps)) + .Select(w => w.Jitter)) + .ToList(); + if (witnesses.Count == 0) + { + grades.Add(baseGrade); + continue; + } + var effective = within.HasValue ? Math.Min(within.Value, witnesses.Min()) : witnesses.Min(); + grades.Add(GradeAsn(series, congestionEvents, jitterFloorMs, accessBaselineRtt, internetMedianDeltaMs, + jitterOverrideMs: effective)); + } + return grades; + } + private List GradeIspHops( List ispHopSeries, List transitSeries, @@ -1159,7 +1255,13 @@ private static List AggregateIspAsns(List hopGrades, return result; } - private IspTargetHealth BuildIspTargetHealth(AsnSeries series, string? firstHopTargetId, List hopGrades, double winsorPercentile) + /// + /// A hop is suggested for disable when its jitter score is at or below this - i.e. jitter is a + /// real deficit, not noise. Combined with "off the traced path" to avoid nagging on clean hops. + /// + private const int DisableSuggestMaxJitterScore = 70; + + private IspTargetHealth BuildIspTargetHealth(AsnSeries series, string? firstHopTargetId, List hopGrades, double winsorPercentile, IReadOnlySet notTracedTargetIds) { var rtts = series.Samples.Where(s => s.RttAvgMs.HasValue).Select(s => s.RttAvgMs!.Value).ToList(); var jitters = series.Samples.Select(s => s.EffectiveJitterMs).Where(j => j.HasValue).Select(j => j!.Value).ToList(); @@ -1169,6 +1271,8 @@ private IspTargetHealth BuildIspTargetHealth(AsnSeries series, string? firstHopT // Jitter comes from the grade (the effective/absolved value the hop is scored on), so // the row matches the grade beside it. Fall back to the hop's own raw P95 when ungraded. var rawP95 = jitters.Count > 0 ? SeriesStats.Percentile(jitters, 0.95) : null; + var isGraded = targetId == firstHopTargetId; + var notTraced = notTracedTargetIds.Contains(targetId); return new IspTargetHealth { TargetId = targetId, @@ -1180,7 +1284,11 @@ private IspTargetHealth BuildIspTargetHealth(AsnSeries series, string? firstHopT LossPct = losses.Count > 0 ? losses.Average() : null, OverallScore = grade?.OverallScore, ReachDeltaMs = grade?.ReachDeltaMs, - IsGradedHop = targetId == firstHopTargetId + IsGradedHop = isGraded, + NotOnTracedPath = notTraced, + // Off the traced path AND its jitter is a real deficit - the "why am I still scoring this?" + // candidate. Never the graded nearest hop. + SuggestDisable = notTraced && !isGraded && grade?.JitterScore is int js && js <= DisableSuggestMaxJitterScore }; } @@ -1206,21 +1314,57 @@ private static IspScoreDimension BuildDimension(string name, double weight, List return new IspScoreDimension { Name = name, Score = score, Weight = weight, Factors = factors }; } + /// + /// Weight below which a graded transit ASN never falls: the least-involved transit still counts + /// at 25% relative to the most-involved (a 4x cap on the spread), so a bad-but-minor transit is + /// de-weighted but never escapes accountability. Arm 4. + /// + private const double TransitInvolvementFloor = 0.25; + + /// Neutral baseline the uninvolved fraction of a transit ASN's score blends toward: a + /// transit carrying none of your hosts can't hurt Transit Health (it contributes ~100). + private const double TransitNeutralBaseline = 100.0; + + /// + /// Builds a transit-style ASN dimension. Each ASN carries its + /// (0.25-1.0, set upstream from internet-host involvement); its effective contribution is + /// weight * own + (1 - weight) * , and the dimension score is + /// the involvement-weighted average of those - so the transit you actually use dominates and a + /// side-path transit's problems don't drag the number. When no ASN has involvement set (no + /// attribution), it's the plain average and no fraction icon is shown. + /// private static IspScoreDimension BuildAsnDimension(string name, double weight, List asns) { var factors = asns.Select(a => new IspScoreFactor { Name = string.IsNullOrEmpty(a.AsnName) ? $"AS{a.AsnNumber}" : a.AsnName, Score = a.OverallScore, - Weight = 1.0, + Weight = a.InvolvementWeight ?? 1.0, ValueText = a.MeanRttMs.HasValue ? FormatMsCoarse(a.MeanRttMs.Value) : null, Description = a.CongestionEventCount > 0 ? $"{a.CongestionEventCount} congestion event{(a.CongestionEventCount == 1 ? "" : "s")} in the window." - : null + : null, + InvolvementTooltip = a.InvolvementTooltip }).ToList(); var scored = asns.Where(a => a.OverallScore.HasValue).ToList(); - int? score = scored.Count > 0 ? (int)Math.Round(scored.Average(a => a.OverallScore!.Value)) : null; + int? score; + if (scored.Count == 0) + score = null; + else if (scored.All(a => a.InvolvementWeight is null)) + score = (int)Math.Round(scored.Average(a => a.OverallScore!.Value)); + else + { + var wsum = scored.Sum(a => a.InvolvementWeight ?? 1.0); + score = wsum > 0 + ? (int)Math.Round(scored.Sum(a => + { + var w = a.InvolvementWeight ?? 1.0; + var effective = w * a.OverallScore!.Value + (1 - w) * TransitNeutralBaseline; + return w * effective; + }) / wsum) + : (int)Math.Round(scored.Average(a => a.OverallScore!.Value)); + } return new IspScoreDimension { Name = name, Score = score, Weight = weight, Factors = factors }; } diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthService.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthService.cs index c4ec6b0eb..ab7b591b4 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthService.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthService.cs @@ -1042,6 +1042,11 @@ double OverlapSeconds(DateTime s, DateTime e) => blackoutSpans.Sum(b => SmartQueuesEnabled = smartQueuesEnabled, AdaptiveSqmEnabled = adaptiveSqmEnabled, HopOrderKnown = hopOrderKnown, + // Hops with a discovery row but HopNumber 0 answered pings yet never landed in a trace + // (OLT/CMTS ICMP-deprioritization); only meaningful once we have trace data at all. + NotTracedTargetIds = hopOrderKnown + ? hopNumberByTargetId.Where(kv => kv.Value == 0).Select(kv => kv.Key).ToHashSet(StringComparer.OrdinalIgnoreCase) + : new HashSet(StringComparer.OrdinalIgnoreCase), LoadExclusionWindows = loadExclusions, PhysicalLink = physical.Input }; diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/UpstreamTracerService.cs b/src/NetworkOptimizer.Web/Services/Monitoring/UpstreamTracerService.cs index 127303787..d8ecf1534 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/UpstreamTracerService.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/UpstreamTracerService.cs @@ -1317,8 +1317,9 @@ private async Task TraceTransitAsnsAsync(CancellationToken ct) /// flipped in either direction: an existing enabled row keeps covering its /// cluster (no second pick is added beside it), and an existing disabled row is /// never re-enabled - a disabled row can be a flaky-target verdict, not just an - /// old default. ASNs with no gate-clearing hop end up with nothing enabled, which - /// is what lets the downstream witness/fallback injection step in. + /// old default. An ASN with no gate-clearing hop ends up with nothing enabled here; + /// that is fine, because a curated transit ASN (e.g. Level 3) still gets its anycast + /// witness attached downstream regardless (InjectTransitWitnessesAsync). /// internal static void ApplyTransitClumpSelection(IEnumerable candidates) { @@ -1367,8 +1368,8 @@ internal static void ApplyTransitClumpSelection(IEnumerable // Reachability gate: a candidate must answer enough pings in a short rapid burst (200 ms // spacing) to be auto-selected, so flaky, ICMP-deprioritized routers don't get monitored - and - // so the Level 3 transit witness (4.2.2.2) is injected exactly when no real AS3356 router clears - // the gate. We always send 3; the required successes depend on the connection's access medium + // so a curated transit witness (e.g. Level 3 4.2.2.2) must itself clear the gate before it is + // attached. We always send 3; the required successes depend on the connection's access medium // (Item B): air-interface mediums (WISP, cellular) and the unconfigured Unknown case allow one // dropped reply (2/3); stable wired/fiber/LEO mediums demand 3/3. private const int ReachabilityPingCount = 3; @@ -1432,12 +1433,12 @@ private async Task VerifyReachabilityAsync(CancellationToken ct) // With verified RTTs in hand, refine the provisional per-clump picks: enable // the lowest-RTT hop that cleared the gate in each of an ASN's clumps (near // ingress + far egress), instead of blindly keeping the lowest hop number. - // Runs before witness injection so an ASN that ends up with a selection - // doesn't also get a witness. ApplyTransitClumpSelection(State.TransitAsns); - // Item A: if Level 3 (AS3356) is on the path but no AS3356 router cleared the gate, inject - // 4.2.2.2 as a transit witness so ISP Health still has Lumen transit data. + // Item A: whenever a curated transit ASN (e.g. Level 3 / AS3356) is near-transit, attach its + // anycast witness (4.2.2.2) as a transit target - alongside any real routers, not only as a + // fallback - so ISP Health always has a stable Lumen transit anchor even when the real hops + // vary POP-to-POP across runs or start deprioritizing ICMP. await InjectTransitWitnessesAsync(minSuccesses, ct); // If the access ASN is one we have curated endpoints for and none of its first-mile @@ -1450,9 +1451,12 @@ private async Task VerifyReachabilityAsync(CancellationToken ct) } // Item A: anycast DNS witnesses for transit ASNs whose routers commonly ICMP-deprioritize or - // hide behind L2-transparent infra. Used only when the ASN is on the path but no router clears - // the reachability gate. The endpoint is anycast (nearest edge), hence the "transit witness" - // label. Extend this table to add witnesses for other transit ASNs (e.g. AS7018 AT&T). + // hide behind L2-transparent infra. Attached whenever the ASN is genuinely near-transit - not + // only as a fallback - so the tier always has a stable anchor even when real routers respond + // (they vary POP-to-POP across runs and can start dropping ICMP). The endpoint is anycast + // (nearest edge), hence the "transit witness" label; it hits a more distant POP than the closest + // real hop, which is why the real hops are kept, never replaced. Extend this table to add + // witnesses for other transit ASNs (e.g. AS7018 AT&T). private static readonly (int Asn, string Address, string Name, string Label)[] TransitWitnesses = { (3356, "4.2.2.2", "Level 3", "Level 3 DNS (transit witness)") @@ -1598,8 +1602,10 @@ private async Task InjectTransitWitnessesAsync(int minSuccesses, CancellationTok if (!_nearTransitAsns.Contains(asn)) continue; // And not when this tier-1 only ever sits above another tier-1 (core peering). if (_excludedTier1Asns.Contains(asn)) continue; - // Skip if any real router in this ASN already cleared the gate, or the witness exists. - if (State.TransitAsns.Any(t => t.AsnNumber == asn && t.Enabled && !t.Unreachable)) continue; + // Skip only if this witness address is already a candidate this run. A real router in the + // same ASN no longer suppresses the witness - the anycast anchor is attached alongside it + // (the real hops are usually a closer POP, so both are kept). Same-address dedup on write + // (UpsertTransitTargetAsync) folds this into any hand-added row at the same address. if (State.TransitAsns.Any(t => string.Equals(t.HopAddress, address, StringComparison.OrdinalIgnoreCase))) continue; // The witness must itself clear the gate before we enable it. @@ -1624,8 +1630,18 @@ private async Task InjectTransitWitnessesAsync(int minSuccesses, CancellationTok VerifiedRttMs = result.RttAvgMs, Enabled = true }); - _logger.LogInformation("Injected transit witness {Address} (AS{Asn} {Name}) - no reachable {Name} router", - address, asn, name, name); + _logger.LogInformation("Attached transit witness {Address} (AS{Asn} {Name}) - near-transit anchor", + address, asn, name); + + // Trace the witness so PersistHopOrderAsync records the real transit hops that precede + // it as its ancestry. Without this the anycast endpoint is ping-only, lands in + // UpstreamDiscoveries with no ancestors, and fails FarClusterRoutesThroughNear - so ISP + // Health can never use the witness's clean end-to-end jitter to absolve a jittery near + // hop it routes through. The proof is honest: ancestry comes from 4.2.2.2's actual path, + // so an absolve only happens when that path genuinely traverses the monitored hop; if + // the anycast lands on a different Level 3 POP there is no overlap and no absolve. + var (_, witnessTrace) = await TraceOneAsync(new TraceEndpoint(name, address), ProbeMode.Icmp, ct); + _lastTraces.Add(witnessTrace); } } diff --git a/src/NetworkOptimizer.Web/wwwroot/css/app.css b/src/NetworkOptimizer.Web/wwwroot/css/app.css index 3551fad00..ecca1f35a 100644 --- a/src/NetworkOptimizer.Web/wwwroot/css/app.css +++ b/src/NetworkOptimizer.Web/wwwroot/css/app.css @@ -16750,6 +16750,29 @@ a.isp-health-hero-speed:hover { color: var(--text-primary); } +.isp-factor-name-row { + display: flex; + align-items: center; + gap: 0.3rem; + min-width: 0; +} + +.isp-asn-name-row { + display: flex; + align-items: center; + gap: 0.3rem; +} + +.isp-involvement-pie { + display: inline-flex; + align-items: center; + flex-shrink: 0; +} + +.isp-involvement-pie svg { + display: block; +} + .isp-factor-value { font-size: 0.75rem; color: var(--text-secondary); @@ -17271,16 +17294,31 @@ a.isp-outage-ack:hover { margin-top: 0.45rem; } -.isp-target-graded { - margin-left: 0.35rem; +.isp-target-badge { + margin-left: 0.5rem; padding: 0.05rem 0.4rem; border-radius: 4px; - background: rgba(5, 80, 181, 0.25); - color: var(--primary-hover); font-size: 0.62rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.04em; + vertical-align: middle; +} + +.isp-target-graded { + background: rgba(5, 80, 181, 0.25); + color: var(--primary-hover); +} + +.isp-target-disable-hint { + border: none; + background: rgba(231, 150, 19, 0.2); + color: var(--warning-color); + cursor: pointer; +} + +.isp-target-disable-hint:hover { + background: rgba(231, 150, 19, 0.32); } .isp-target-stat { diff --git a/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs b/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs index 073cd7de7..d2b05ec9c 100644 --- a/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs @@ -40,7 +40,8 @@ private static IspHealthInputs BuildInputs( bool lineIdle = false, bool hopOrderKnown = false, List? outages = null, - TimeSpan? scoreWindow = null) + TimeSpan? scoreWindow = null, + HashSet? notTracedTargetIds = null) { // lineIdle: a near-zero, flat WAN with no load bursts (~0% average load), for // exercising the load-calibrated packet-loss ceiling at the idle end. @@ -82,6 +83,7 @@ private static IspHealthInputs BuildInputs( CongestionEvents = congestion ?? new List(), SmartQueuesEnabled = smartQueuesEnabled, HopOrderKnown = hopOrderKnown, + NotTracedTargetIds = notTracedTargetIds ?? new HashSet(), Outages = outages ?? new List() }; } @@ -1009,6 +1011,159 @@ public void A_clean_farther_cluster_absolves_false_near_jitter() graded.RawJitterMs.Should().BeApproximately(4.0, 0.1, "the raw near reading is kept for the tooltip"); } + [Fact] + public void A_clean_destination_absolves_a_transit_asn_it_routes_through() + { + // A transit ASN shows high self-jitter (ICMP deprioritization) with no farther cluster of + // its own to disprove it. A monitored destination reached THROUGH the ASN (its hop is in the + // destination's ancestor set) is clean end-to-end - a hard upper bound on the ASN's true + // jitter - so it absolves the transit ASN. Arm A. Divergent ancestry = no absolve (strict). + AsnSeries Transit() => new() + { + AsnNumber = 64500, + AsnName = "TransitOne", + TargetIds = { "transit-jittery" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 10, 4.0), + HopIps = { "20.0.0.1" } + }; + var cleanDest = new AsnSeries + { + AsnNumber = 64512, + AsnName = "Destination", + TargetIds = { "dest-clean" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 13, 0.4), + HopIps = { "30.0.0.1" }, + AncestorIps = { "20.0.0.1" } // reached through the transit ASN's hop + }; + var divergentDest = new AsnSeries + { + AsnNumber = 64512, + AsnName = "Destination", + TargetIds = { "dest-clean" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 13, 0.4), + HopIps = { "30.0.0.1" }, + AncestorIps = { "10.9.9.9" } // a different hop - does not route through our transit + }; + + var absolved = new IspHealthScorer(Options).Score( + BuildInputs(transit: new List { Transit() }, + destinations: new List { cleanDest }, hopOrderKnown: true), Gpon).TransitAsns.Single(); + var notAbsolved = new IspHealthScorer(Options).Score( + BuildInputs(transit: new List { Transit() }, + destinations: new List { divergentDest }, hopOrderKnown: true), Gpon).TransitAsns.Single(); + + absolved.JitterScore.Should().BeGreaterThan(notAbsolved.JitterScore!.Value, + "a clean destination routing through the transit ASN proves its jitter is an ICMP artifact"); + absolved.JitterAssimilated.Should().BeTrue("the clean destination pulled the jitter down"); + } + + [Fact] + public void Transit_health_weights_asns_by_internet_host_involvement() + { + // Two transit ASNs, one clean and one jittery. The jittery one carries NO monitored internet + // host, so involvement weighting drops it to the 25% floor and the dimension leans on the + // clean, host-carrying ASN - scoring higher than the plain average an install with no + // attribution would get. Arm 4. The destination routes only through the clean ASN, so Arm A + // can't rescue the jittery one's jitter (isolating the weighting effect). + List Transits() => new() + { + new AsnSeries + { + AsnNumber = 64500, AsnName = "CleanTransit", + TargetIds = { "transit-clean" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 10, 0.4), + HopIps = { "20.0.0.1" } + }, + new AsnSeries + { + AsnNumber = 64600, AsnName = "JitteryTransit", + TargetIds = { "transit-jittery" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 12, 4.0), + HopIps = { "21.0.0.1" } + } + }; + var dest = new AsnSeries + { + AsnNumber = 64512, AsnName = "Destination", + TargetIds = { "dest-clean" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 13, 0.4), + HopIps = { "30.0.0.1" }, + AncestorIps = { "20.0.0.1" } // routes through the clean transit only + }; + + var weighted = new IspHealthScorer(Options).Score( + BuildInputs(transit: Transits(), destinations: new List { dest }, hopOrderKnown: true), Gpon) + .TransitDimension; + // Baseline: no attribution, so both ASNs weigh equally - the plain average, no tooltips. + var equal = new IspHealthScorer(Options).Score( + BuildInputs(transit: Transits(), hopOrderKnown: true), Gpon) + .TransitDimension; + + weighted.Score.Should().BeGreaterThan(equal.Score!.Value, + "down-weighting the jittery, host-less transit lifts the dimension toward the clean, host-carrying one"); + weighted.Factors.Single(f => f.Name == "CleanTransit").InvolvementTooltip.Should().Contain("100% weight"); + weighted.Factors.Single(f => f.Name == "JitteryTransit").InvolvementTooltip.Should().Contain("25%"); + equal.Factors.Should().OnlyContain(f => f.InvolvementTooltip == null, + "with no attributable host there is no involvement to differentiate"); + } + + [Fact] + public void Off_path_jittery_isp_hop_is_flagged_for_disable() + { + // A hop that answers pings but appears on no trace (an OLT that ICMP-deprioritizes traceroute) + // with high jitter is flagged SuggestDisable. The graded on-path hop never is. + var graded = new AsnSeries + { + AsnNumber = 64496, AsnName = "ISP", TargetIds = { "isp-near" }, RoleTargetIds = { "isp-near" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 2.0, 0.3), HopIps = { "10.0.0.1" } + }; + var offPathJittery = new AsnSeries + { + AsnNumber = 64496, AsnName = "ISP", TargetIds = { "isp-olt" }, RoleTargetIds = { "isp-olt" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 4.0, 6.0), HopIps = { "10.0.0.9" } + }; + var hops = new List { graded, offPathJittery }; + + var report = new IspHealthScorer(Options).Score( + BuildInputs(ispAsn: hops, ispTargets: hops, firstHopTargetId: "isp-near", + hopOrderKnown: true, notTracedTargetIds: new HashSet { "isp-olt" }), Gpon); + + var olt = report.IspTargets.Single(t => t.TargetId == "isp-olt"); + olt.NotOnTracedPath.Should().BeTrue(); + olt.SuggestDisable.Should().BeTrue("off the traced path and dragging its score with jitter"); + report.IspTargets.Single(t => t.TargetId == "isp-near").SuggestDisable.Should().BeFalse( + "the graded on-path hop is never suggested for disable"); + } + + [Fact] + public void Transit_asns_with_no_attributable_hosts_are_floored_and_labeled() + { + // A fully-peered site: destinations reach the internet directly, so their (complete) ancestry + // crosses no transit and every transit ASN carries zero monitored hosts. With attribution + // available (hop order + destinations), that is a TRUE zero - each transit is floored at 25% + // and labeled, not left at the equal-weight "unknown" fallback. Arm 4. + List Transits() => new() + { + new AsnSeries { AsnNumber = 64500, AsnName = "TransitA", TargetIds = { "ta" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 10, 0.5), HopIps = { "20.0.0.1" } }, + new AsnSeries { AsnNumber = 64600, AsnName = "TransitB", TargetIds = { "tb" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 12, 0.5), HopIps = { "21.0.0.1" } } + }; + var peeredDest = new AsnSeries + { + AsnNumber = 64512, AsnName = "Destination", TargetIds = { "dest" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 8, 0.4), HopIps = { "30.0.0.1" }, + AncestorIps = { "9.9.9.9" } // routes through neither transit (peered) + }; + + var dim = new IspHealthScorer(Options).Score( + BuildInputs(transit: Transits(), destinations: new List { peeredDest }, hopOrderKnown: true), Gpon) + .TransitDimension; + + dim.Factors.Should().OnlyContain(f => f.InvolvementTooltip != null && f.InvolvementTooltip.Contains("25%"), + "complete ancestry showing zero transit involvement floors and labels every transit ASN"); + } + [Fact] public void Without_hop_order_a_transit_asn_is_graded_on_its_near_cluster() { From a8b1079780e2fb15e992d775a07357682546671d Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:50:12 -0500 Subject: [PATCH 05/23] ONT Device Monitoring: Network Optimizer Custom (HTTP JSON) provider with SFP module attachment (#1030) * ONT provider: Network Optimizer Custom (HTTP JSON) with SFP module attachment - New netopt-custom ONT provider polling a vendor-neutral PON stats JSON endpoint (contract doc in docs/features/netopt-custom-pon-contract.md) - Attach-to-SFP-module mode: polled on the gateway SFP collection cycle and merged onto that module's sfp measurement; pon_link_status / fec_errors / bip_errors reuse the ont measurement's field names and encodings - Standalone mode feeds the ont measurement and ONT alerts like any provider - Settings - ONT Device Monitoring: provider entry + Attach to SFP Module selector (OntConfigurations.AttachedSfpId migration) - SFP Stats: conditional PON errors / GEM frames / host link delta charts and PLOAM details table for supplemented modules - ONT Stats: FEC / BIP per-interval delta chart (all providers reporting the counters) * ISP Health: uniform hop badge height and centering * Attached ONT configs: exclude from standalone ONT UI (Dashboard, ONT Stats, ISP Health) An ONT config attached to an SFP module surfaces as PON data on that module (SFP Stats), so it must not also appear as its own ONT device. Add GetStandaloneConfigsAsync (filters AttachedSfpId) and use it for the ONT Stats tab, Dashboard ONT card, and ONT chart endpoint; exclude attached configs from ISP Health PON candidates too. * SFP ONT card: show PON status and absolute BIP/FEC/drops when available Cache the latest PON supplement (link status + cumulative BIP/FEC/GEM-dropped) in SfpLiveStats during the SFP poll, and surface it on the SFP ONT Connection card: PON Status in the Connection box (same format as the external ONT card) and absolute BIP/FEC/Drops in the Stats box, each shown only when present. * SFP ONT card: combine BIP/FEC/Drops into one row to save space * SFP ONT card: reorder PON stats, adaptive FEC/HEC, even columns - PON Status moves directly below Link Type in the Connection box - BIP/FEC/Drops error counts move into the Connection box - Middle error slot is adaptive: shows FEC when payload FEC is enabled, HEC (the always-on header-error signal) when FEC is disabled, label flips to match - SFP Voltage moves to the Stats column only when a PON supplement is present, keeping the two columns even * SFP/ONT charts: lazy-mount PON + FEC/BIP charts, filter-aware PON section - Don't create the PON charts (SFP Stats) or the FEC/BIP errors chart (ONT Stats) until a module actually reports supplemental data. Setups without supplemental polling - the large majority - never mount instances they'd never see. - PON section and its detail table now follow the filter: shown only while a PON-capable module is selected, table lists just the selected modules. * SFP Stats: fix filter-chip breakage and PON chart sizing; drop TX idle - PON section refresh is now fire-and-forget off updateVisibility, so an ApexCharts error can never abort the synchronous chip re-render (was leaving chip highlight desynced and blocking re-selection) - Show the PON section before mounting/updating its charts so ApexCharts sizes them correctly instead of loading blank until the next tick - Rebuild PON series from only the visible modules each refresh (no per-series show/hide bookkeeping) - Drop TX idle from the GEM chart - it's an upstream-utilization indicator, not a health signal, and dominated the scale * Review fixes: per-endpoint poll gate, awaited PON alert eval, resilient chart refresh - NetOptCustomPonOntProvider: key the request gate per (host, port) instead of one global semaphore, so distinct endpoints/sites poll in parallel - CollectPonSupplementalAsync: await the PON alert evaluation inside its own try/catch (was fire-and-forget; unobserved exceptions escaped the poll catch) - sfp/ont charts: guard the PON/errors chart refresh in loadAndUpdate so a chart failure can't abort badge/stats-table rendering * ONT alerts: add BIP and adaptive HEC spike alerting - OntAlertEvaluator now evaluates BIP (always-on) and, when payload FEC is disabled, HEC in place of FEC - mirroring the SFP ONT card's adaptive display. Shared per-poll spike helper, reset-guarded like the existing FEC check. - Seed ONT: BIP Error Spike and ONT: HEC Error Spike rules (disabled by default). - Attaching augmented PON polling to an SFP ONT enables the BIP/HEC rules immediately (AlertRuleAutoEnable.EnablePatternsAsync); existing ONT users also get them via the standard freshly-seeded startup pass. Closes the gap where a FEC-disabled attached ONT had no error-counter alerting. - Feed BIP from the standalone ONT path and BIP/HEC/FEC-enable from the attached path. * ONT alerts: tune BIP/HEC thresholds, make BIP FEC-state-aware - HEC spike threshold 100 -> 10 (uncorrectable header errors; healthy reads ~0) - BIP spike threshold is now FEC-state-aware: strict 25/poll when payload FEC is disabled (every BIP is uncorrected data loss), relaxed 1000/poll when FEC is on or unknown (BIP counts pre-FEC line errors FEC corrects, so a healthy link at the normal operating point reads hundreds/poll) * ONT alerts: deep-link to the triggering ONT Alerts now carry a per-ONT SourceUrl instead of a fixed /monitoring?tab=ont: standalone ONTs link to the ONT tab focused on that device (?ont=id), attached SFP ONTs link to SFP Stats focused on that module (?sfp=mac:port), so clicking an alert lands on the tab that actually shows it. * TODO: note wrap-aware PON counter deltas as a low-priority follow-up * SFP Stats: group PON error charts, GEM frames last Reorder the PON section to PON Errors -> Host Link Errors -> GEM Frames so the two error charts sit adjacent (faster upstream-vs-host fault localization, consistent units) with throughput last. --- TODO.md | 14 + docs/features/netopt-custom-pon-contract.md | 156 + .../DefaultAlertRules.cs | 22 + .../Models/PonSupplementalStats.cs | 117 + .../Providers/ISfpSupplementalOntProvider.cs | 22 + ...0719120000_AddOntAttachedSfpId.Designer.cs | 3236 +++++++++++++++++ .../20260719120000_AddOntAttachedSfpId.cs | 30 + .../NetworkOptimizerDbContextModelSnapshot.cs | 3 + .../Models/OntConfiguration.cs | 9 + .../Repositories/OntRepository.cs | 1 + .../Services/MonitoringInfluxClient.cs | 160 + .../Components/Pages/Monitoring.razor | 26 +- .../Components/Pages/Settings.razor | 50 + .../Components/Shared/OntStatsPanel.razor | 52 +- .../Endpoints/OntChartEndpoints.cs | 31 +- .../Endpoints/SfpChartEndpoints.cs | 52 +- src/NetworkOptimizer.Web/Program.cs | 1 + .../Services/AlertRuleAutoEnable.cs | 30 + .../IspHealth/PhysicalLinkResolver.cs | 4 +- .../Services/Monitoring/OntAlertEvaluator.cs | 98 +- .../Services/Monitoring/PonThresholds.cs | 16 +- .../Services/MonitoringCollectionAgent.cs | 150 +- .../Services/MonitoringLiveStats.cs | 60 +- .../Services/OntMonitorService.cs | 36 +- .../NetOptCustomPonOntProvider.cs | 419 +++ src/NetworkOptimizer.Web/wwwroot/css/app.css | 16 +- .../wwwroot/js/ont-charts.js | 56 + .../wwwroot/js/sfp-charts.js | 124 + .../NetOptCustomPonOntProviderTests.cs | 158 + .../OntAlertEvaluatorTests.cs | 76 + 30 files changed, 5179 insertions(+), 46 deletions(-) create mode 100644 docs/features/netopt-custom-pon-contract.md create mode 100644 src/NetworkOptimizer.Core/Models/PonSupplementalStats.cs create mode 100644 src/NetworkOptimizer.Monitoring/Providers/ISfpSupplementalOntProvider.cs create mode 100644 src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.Designer.cs create mode 100644 src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.cs create mode 100644 src/NetworkOptimizer.Web/Services/OntProviders/NetOptCustomPonOntProvider.cs create mode 100644 tests/NetworkOptimizer.Web.Tests/NetOptCustomPonOntProviderTests.cs diff --git a/TODO.md b/TODO.md index 4825dad03..955257834 100644 --- a/TODO.md +++ b/TODO.md @@ -321,6 +321,20 @@ The following were implemented in the WiFi Optimizer feature: ## Monitoring +### PON supplemental counters: wrap-aware deltas (SFP Stats) +The augmented PON provider's frame/allocation counters (GEM tx/rx, LAN frames, allocations) are 32-bit +on the ONT and wrap (~4.29B unsigned, or ~2.15B if signed). Every path handles the wrap **safely** +today - the chart delta guard (`cur >= prev ? cur - prev : null`), the alert spike check +(`if (delta < 0) delta = 0`), and ISP Health's positive-increment total all treat a wrap like a +counter reset. The only cost is a single dropped data point on the frame charts at each wrap, which on +a busy link can be ~hourly for the high-rate counters. + +Follow-up (low priority): reconstruct the true delta across a wrap instead of gapping it. Use +`sfp_uptime_s` to distinguish a wrap (uptime kept climbing) from a real reset/reboot (uptime dropped), +then add the modular span. Blocker: we'd have to know the counter width (2^31 vs 2^32) per field, and a +wrong guess produces a garbage spike - so the safe null-gap stays the default until the widths are +confirmed. Not worth it for a rare one-point gap; revisit if the frame charts read as too sparse. + ### Upstream path discovery: DynamoDB regional endpoints for transit trace exposure (Setup tab) Transit Health involvement weighting and destination->transit jitter absolution both key off which monitored internet targets a transit ASN provably carries (trace ancestry). The current CDN/DNS diff --git a/docs/features/netopt-custom-pon-contract.md b/docs/features/netopt-custom-pon-contract.md new file mode 100644 index 000000000..434c3adbc --- /dev/null +++ b/docs/features/netopt-custom-pon-contract.md @@ -0,0 +1,156 @@ +# Network Optimizer Custom PON Stats - JSON Contract v1 + +The **Network Optimizer Custom (HTTP JSON)** ONT provider polls a plain HTTP +endpoint that returns PON-layer statistics as JSON. The contract is +vendor-neutral (field semantics follow ITU-T G.984.x / G.9807.1 terminology), so +anyone can implement it for their ONT hardware - a GPON/XGS-PON SFP stick, a +full ONT, or a small relay script on the gateway that gathers stats from the +device and serves them. + +Configure it under Settings - ONT Device Monitoring with provider +"Network Optimizer Custom (HTTP JSON)". Two modes: + +- **Standalone**: polled like any other ONT provider; PON state and FEC/BIP + counters flow to the ONT Stats tab and ONT alerts. +- **Attached to a monitored SFP module** (the "Attach to SFP Module" setting): + polled on the gateway SFP collection cycle instead, with the full metric set + merged into that module's `sfp` measurement and charted on the SFP Stats tab. + Use this when the ONT *is* the SFP module in your gateway, so its DDM optics + readings and its PON internals form one time series. + +## Endpoint semantics + +- `GET http://:/` (any path is ignored by the reference + implementation; the provider requests `/`). Default port: **10012**. +- Response: `Content-Type: application/json`, UTF-8, a single JSON object. +- No authentication. Serve it on a management/LAN interface only. +- The provider allows up to **15 seconds** per request and never issues + concurrent requests, so an implementation may gather stats on demand (e.g. an + SSH round-trip into an SFP stick) and may be a one-shot listener loop. +- Stats should be gathered fresh per request (or cached only briefly). + +### Failure shape + +When the implementation cannot reach the underlying device, return HTTP 200 +with an error object instead of stats: + +```json +{ + "error": "sfp_unreachable", + "message": "Could not reach SFP at 169.254.1.1:30007" +} +``` + +A non-empty `error` marks the poll failed; `message` is a human-readable +detail shown in the UI. Any transport failure (refused, timeout, non-JSON) is +treated the same way. + +## Payload + +Every section and every field is **optional** - serve what the hardware +exposes; absent values are simply not recorded. All counters are **cumulative +since ONT boot** (Network Optimizer derives per-interval deltas and uses +`sfp_uptime_s` to recognize counter resets). Numbers may be JSON numbers or +numeric strings; 64-bit counters are expected. + +```json +{ + "lan": { + "mode": 15, // host-side PHY mode (raw device enum, informational) + "link_status": 5, // host (module-to-gateway) link state, raw enum + "phy_duplex": 1 // raw enum, informational + }, + "lan_counters": { // host-link MAC counters (module <-> gateway) + "tx_frames": 671630919, + "rx_frames": 850075595, + "tx_drop_events": 0, + "rx_fcs_err": 0, // FCS/checksum errors - host-link signal integrity + "buffer_overflow": 0 + }, + "ploam": { + "curr_state": 5, // ITU-T activation state, numeric: 1-7 = O1-O7 (5 = O5 Operation) + "previous_state": 4, // state before the last transition + "elapsed_msec": 12345 // ms in the current state (uint32 on many devices; wraps ~49.7 days) + }, + "gtc_status": { + "ds_state": 3, // downstream GTC sync state (raw enum; 3 = sync) + "onu_id": 49, // ONU ID assigned by the OLT; changes on re-ranging + "ds_fec_enable": 0, // OLT profile: downstream FEC on/off (0/1) + "us_fec_enable": 0, // OLT profile: upstream FEC on/off (0/1) + "onu_response_time": 34992 // raw ranging response time / equalization delay + }, + "gtc_counters": { + "bip": 0, // BIP parity errors (0 on a healthy link) + "hec_error_corr": 0, // GTC header errors, corrected + "hec_error_uncorr": 1, // GTC header errors, uncorrectable + "bwmap_error_corr": 0, // upstream bandwidth-map errors, corrected + "bwmap_error_uncorr": 0, // upstream bandwidth-map errors, uncorrectable + "fec_error_corr": 0, // corrected FEC bit errors (informational) + "fec_words_corr": 0, // corrected FEC codewords (early warning) + "fec_words_uncorr": 0, // UNCORRECTABLE FEC codewords - the data-loss signal + "fec_words_total": 0, + "fec_seconds": 0, + "tx_gem_frames_total": 848555332, // (X)GEM frames sent upstream + "tx_gem_bytes_total": 1774937051, // (often 32-bit; unreliable at line rate) + "tx_gem_idle_frames_total": 1076427353, // idle frames = granted-but-unused upstream capacity + "rx_gem_frames_total": 682156109, + "rx_gem_bytes_total": 0, + "rx_gem_frames_dropped": 1, + "omci_drop": 0, + "drop": 1, + "rx_oversized_frames": 0, + "allocations_total": 1629046208, // upstream grants received from the OLT + "allocations_lost": 20 // missed grants - scheduling/resync indicator + }, + "gpe_pon": { // PON-side bridge port counters (packet engine) + "ibp_good": 670798566, "ibp_discard": 0, // ingress good/discarded + "ebp_good": 848843235, "ebp_discard": 0, // egress good/discarded + "learning_discard": 0 // MAC-learning-limit discards + }, + "gpe_lan": { // host-side bridge port, same fields as gpe_pon + "ibp_good": 848843244, "ibp_discard": 0, + "ebp_good": 670798569, "ebp_discard": 0, + "learning_discard": 0 + }, + "sfp_uptime_s": 358825 // seconds since the ONT module booted (counter-reset anchor) +} +``` + +## What Network Optimizer records + +Not every field is stored. The curated set, and how the standard concepts map +onto Network Optimizer's existing schema: + +| Contract field | Stored as | Notes | +|---|---|---| +| `ploam.curr_state` | `pon_link_status` | same encoding as the ont measurement: `initial`, `standby`, `serial_number`, `ranging`, `operation`, `popup`, `emergency_stop` | +| `ploam.previous_state` | `pon_link_status_prev` | same encoding | +| `ploam.elapsed_msec` | `ploam_elapsed_ms` | | +| `gtc_status.ds_state` | `gtc_ds_state` | | +| `gtc_status.onu_id` | `onu_id` | | +| `gtc_status.ds_fec_enable` / `us_fec_enable` | `ds_fec_enabled` / `us_fec_enabled` | | +| `gtc_status.onu_response_time` | `onu_response_time` | | +| `gtc_counters.bip` | `bip_errors` | standard field | +| `gtc_counters.fec_words_uncorr` | `fec_errors` | standard field: uncorrectable codewords | +| `gtc_counters.fec_words_corr` | `fec_corrected_words` | | +| `gtc_counters.hec_error_corr` / `uncorr` | `hec_corrected` / `hec_uncorrected` | | +| `gtc_counters.bwmap_error_corr` / `uncorr` | `bwmap_corrected` / `bwmap_uncorrected` | | +| `gtc_counters.tx_gem_frames_total` etc. | `gem_tx_frames`, `gem_tx_idle_frames`, `gem_rx_frames`, `gem_rx_dropped` | | +| `gtc_counters.allocations_total` / `lost` | `alloc_total` / `alloc_lost` | | +| `gpe_pon` / `gpe_lan` discards | `gpe_{pon,lan}_{ingress,egress,learning}_discard` | good-frame counters are not stored | +| `lan.link_status` | `lan_link_status` | | +| `lan_counters.*` | `lan_tx_frames`, `lan_rx_frames`, `lan_tx_drop_events`, `lan_rx_fcs_err`, `lan_buffer_overflow` | | +| `sfp_uptime_s` | `sfp_uptime_s` | | + +Not recorded: `lan.mode`, `lan.phy_duplex` (static config), GEM byte counters +(32-bit wrap), `drop` / `omci_drop` / `rx_oversized_frames`, +`fec_error_corr` / `fec_words_total` / `fec_seconds`, and the `gpe_*` good-frame +counters (redundant with `lan_counters` and GEM frame counters). + +## Reference implementation + +The first serving-side implementation is a pair of shell scripts on a UniFi +gateway that SSH into a Lantiq-based GPON SFP stick, run the vendor `onu` CLI +(`ploamsg`, `gtcsg`, `gtctcg`, `gpebptcg`, `lanpsg`, `lantcg`), and serve the +JSON via a one-shot netcat accept loop on port 10012. Any equivalent works - +the provider only sees the HTTP contract above. diff --git a/src/NetworkOptimizer.Alerts/DefaultAlertRules.cs b/src/NetworkOptimizer.Alerts/DefaultAlertRules.cs index 6a9a5ebbe..9131785ad 100644 --- a/src/NetworkOptimizer.Alerts/DefaultAlertRules.cs +++ b/src/NetworkOptimizer.Alerts/DefaultAlertRules.cs @@ -367,6 +367,28 @@ public static List GetDefaults() => CooldownSeconds = 1800 // 30 minutes }, new AlertRule + { + // Uncorrected bit errors; always-on signal, and the primary error alert on a + // link running with payload FEC disabled. Only ONTs whose provider reports BIP trip it. + Name = "ONT: BIP Error Spike", + IsEnabled = false, + EventTypePattern = "ont.bip_errors", + Source = "ont", + MinSeverity = AlertSeverity.Warning, + CooldownSeconds = 1800 // 30 minutes + }, + new AlertRule + { + // Uncorrectable framing-header errors; the live codeword-error signal when payload + // FEC is disabled. Only augmented SFP-ONT polling reports HEC, so it stays inert otherwise. + Name = "ONT: HEC Error Spike", + IsEnabled = false, + EventTypePattern = "ont.hec_errors", + Source = "ont", + MinSeverity = AlertSeverity.Warning, + CooldownSeconds = 1800 // 30 minutes + }, + new AlertRule { // Only ONTs whose provider reports a temperature can trip this; the rest never fire it. Name = "ONT: Temperature High", diff --git a/src/NetworkOptimizer.Core/Models/PonSupplementalStats.cs b/src/NetworkOptimizer.Core/Models/PonSupplementalStats.cs new file mode 100644 index 000000000..966ad3433 --- /dev/null +++ b/src/NetworkOptimizer.Core/Models/PonSupplementalStats.cs @@ -0,0 +1,117 @@ +namespace NetworkOptimizer.Core.Models; + +/// +/// Supplemental PON-layer statistics for an SFP ONT module, fetched from a +/// "Network Optimizer Custom (HTTP JSON)" stats endpoint and merged into the +/// module's sfp measurement during the gateway SFP poll. All counters are +/// cumulative since ONT boot; detects counter resets. +/// PLOAM states are pre-encoded with the same stable strings the ont +/// measurement's pon_link_status field uses (PonLinkStateExtensions.ToInfluxValue). +/// +public class PonSupplementalStats +{ + /// Raw ITU-T PLOAM state number (1-7 = O1-O7) for alert evaluation. + public long? PloamStateRaw { get; set; } + + /// Current PLOAM state, encoded like the ont measurement's pon_link_status ("operation", "popup", ...). + public string? PonLinkStatus { get; set; } + + /// Previous PLOAM state, same encoding as . + public string? PonLinkStatusPrev { get; set; } + + /// Milliseconds spent in the current PLOAM state. uint32 on-device; wraps at ~49.7 days. + public long? PloamElapsedMs { get; set; } + + /// Downstream GTC synchronization state (raw device enum). + public long? GtcDsState { get; set; } + + /// ONU ID assigned by the OLT; changes on re-ranging. + public long? OnuId { get; set; } + + /// Whether downstream FEC is enabled by the OLT profile (0/1). + public long? DsFecEnabled { get; set; } + + /// Whether upstream FEC is enabled by the OLT profile (0/1). + public long? UsFecEnabled { get; set; } + + /// Raw ranging response time reported by the device; changes on re-ranging. + public long? OnuResponseTime { get; set; } + + /// BIP (bit-interleaved parity) errors. Cumulative; 0 on a healthy link. + public long? BipErrors { get; set; } + + /// Uncorrectable FEC codewords - the data-loss signal, matching the ont measurement's fec_errors. + public long? FecErrors { get; set; } + + /// Corrected FEC codewords - benign, early-warning counterpart to . + public long? FecCorrectedWords { get; set; } + + /// GTC header errors, corrected. + public long? HecCorrected { get; set; } + + /// GTC header errors, uncorrectable. + public long? HecUncorrected { get; set; } + + /// Upstream bandwidth-map errors, corrected. + public long? BwmapCorrected { get; set; } + + /// Upstream bandwidth-map errors, uncorrectable. + public long? BwmapUncorrected { get; set; } + + /// GEM frames transmitted upstream (data). + public long? GemTxFrames { get; set; } + + /// Idle GEM frames transmitted upstream - granted capacity that went unused. + public long? GemTxIdleFrames { get; set; } + + /// GEM frames received downstream. + public long? GemRxFrames { get; set; } + + /// GEM frames dropped at reassembly. + public long? GemRxDropped { get; set; } + + /// Upstream bandwidth allocations (grants) received from the OLT. + public long? AllocTotal { get; set; } + + /// Upstream allocations lost - missed grants; scheduling/resync indicator. + public long? AllocLost { get; set; } + + /// PON-side bridge port: ingress frames discarded. + public long? GpePonIngressDiscard { get; set; } + + /// PON-side bridge port: egress frames discarded. + public long? GpePonEgressDiscard { get; set; } + + /// PON-side bridge port: frames discarded by MAC learning limits. + public long? GpePonLearningDiscard { get; set; } + + /// Host-side bridge port: ingress frames discarded. + public long? GpeLanIngressDiscard { get; set; } + + /// Host-side bridge port: egress frames discarded. + public long? GpeLanEgressDiscard { get; set; } + + /// Host-side bridge port: frames discarded by MAC learning limits. + public long? GpeLanLearningDiscard { get; set; } + + /// Host (module-to-gateway) link PHY status, raw device enum. + public long? LanLinkStatus { get; set; } + + /// Frames transmitted on the host link. + public long? LanTxFrames { get; set; } + + /// Frames received on the host link. + public long? LanRxFrames { get; set; } + + /// Transmit drop events on the host link. + public long? LanTxDropEvents { get; set; } + + /// FCS (checksum) errors received on the host link - signal-integrity indicator. + public long? LanRxFcsErrors { get; set; } + + /// Buffer overflows on the host link. + public long? LanBufferOverflow { get; set; } + + /// Seconds since the ONT module booted. Anchors counter-reset detection. + public long? SfpUptimeS { get; set; } +} diff --git a/src/NetworkOptimizer.Monitoring/Providers/ISfpSupplementalOntProvider.cs b/src/NetworkOptimizer.Monitoring/Providers/ISfpSupplementalOntProvider.cs new file mode 100644 index 000000000..f128685d6 --- /dev/null +++ b/src/NetworkOptimizer.Monitoring/Providers/ISfpSupplementalOntProvider.cs @@ -0,0 +1,22 @@ +using NetworkOptimizer.Core.Models; + +namespace NetworkOptimizer.Monitoring.Providers; + +/// +/// An ONT provider that can supplement a gateway-slot SFP ONT module with +/// PON-layer statistics. Configurations attached to a monitored SFP module +/// (OntConfiguration.AttachedSfpId) are polled on the gateway SFP collection +/// cycle via and their metrics merged into +/// that module's sfp measurement, instead of being polled standalone. +/// +public interface ISfpSupplementalOntProvider : IOntProvider +{ + /// + /// Poll the endpoint and return PON-layer supplemental stats. + /// Implementations should log internally and return null on transport + /// or parsing failure; throwing is reserved for programming errors. + /// + Task PollSupplementalAsync( + OntPollContext context, + CancellationToken cancellationToken = default); +} diff --git a/src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.Designer.cs b/src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.Designer.cs new file mode 100644 index 000000000..444e6bc41 --- /dev/null +++ b/src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.Designer.cs @@ -0,0 +1,3236 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using NetworkOptimizer.Storage.Models; + +#nullable disable + +namespace NetworkOptimizer.Storage.Migrations +{ + [DbContext(typeof(NetworkOptimizerDbContext))] + [Migration("20260719120000_AddOntAttachedSfpId")] + partial class AddOntAttachedSfpId + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.7"); + + modelBuilder.Entity("NetworkOptimizer.Alerts.Models.AlertHistoryEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AcknowledgedAt") + .HasColumnType("TEXT"); + + b.Property("ContextJson") + .HasColumnType("TEXT"); + + b.Property("DeliveredToChannels") + .HasColumnType("TEXT"); + + b.Property("DeliveryError") + .HasColumnType("TEXT"); + + b.Property("DeliverySucceeded") + .HasColumnType("INTEGER"); + + b.Property("DeviceId") + .HasColumnType("TEXT"); + + b.Property("DeviceIp") + .HasColumnType("TEXT"); + + b.Property("DeviceName") + .HasColumnType("TEXT"); + + b.Property("EventType") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IncidentId") + .HasColumnType("INTEGER"); + + b.Property("Message") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ResolvedAt") + .HasColumnType("TEXT"); + + b.Property("RuleId") + .HasColumnType("INTEGER"); + + b.Property("Severity") + .HasColumnType("INTEGER"); + + b.Property("Source") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SourceUrl") + .HasColumnType("TEXT"); + + b.Property("Status") + .HasColumnType("INTEGER"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TriggeredAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IncidentId"); + + b.HasIndex("RuleId"); + + b.HasIndex("Status"); + + b.HasIndex("TriggeredAt"); + + b.HasIndex("Source", "TriggeredAt"); + + b.ToTable("AlertHistory", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Alerts.Models.AlertIncident", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AlertCount") + .HasColumnType("INTEGER"); + + b.Property("CorrelationKey") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("FirstTriggeredAt") + .HasColumnType("TEXT"); + + b.Property("LastTriggeredAt") + .HasColumnType("TEXT"); + + b.Property("ResolvedAt") + .HasColumnType("TEXT"); + + b.Property("Severity") + .HasColumnType("INTEGER"); + + b.Property("Status") + .HasColumnType("INTEGER"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CorrelationKey"); + + b.HasIndex("Status"); + + b.ToTable("AlertIncidents", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Alerts.Models.AlertRule", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CooldownSeconds") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DigestOnly") + .HasColumnType("INTEGER"); + + b.Property("EscalationMinutes") + .HasColumnType("INTEGER"); + + b.Property("EscalationSeverity") + .HasColumnType("INTEGER"); + + b.Property("EventTypePattern") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsEnabled") + .HasColumnType("INTEGER"); + + b.Property("MinSeverity") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Source") + .HasColumnType("TEXT"); + + b.Property("TargetDevices") + .HasColumnType("TEXT"); + + b.Property("ThresholdPercent") + .HasColumnType("REAL"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AlertRules", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Alerts.Models.DeliveryChannel", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ChannelType") + .HasColumnType("INTEGER"); + + b.Property("ConfigJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DigestEnabled") + .HasColumnType("INTEGER"); + + b.Property("DigestSchedule") + .HasColumnType("TEXT"); + + b.Property("IsEnabled") + .HasColumnType("INTEGER"); + + b.Property("MinSeverity") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("DeliveryChannels", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Alerts.Models.ScheduledTask", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("CustomEveningHour") + .HasColumnType("INTEGER"); + + b.Property("CustomEveningMinute") + .HasColumnType("INTEGER"); + + b.Property("CustomMorningHour") + .HasColumnType("INTEGER"); + + b.Property("CustomMorningMinute") + .HasColumnType("INTEGER"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("FrequencyMinutes") + .HasColumnType("INTEGER"); + + b.Property("LastErrorMessage") + .HasColumnType("TEXT"); + + b.Property("LastResultSummary") + .HasColumnType("TEXT"); + + b.Property("LastRunAt") + .HasColumnType("TEXT"); + + b.Property("LastStatus") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("NextRunAt") + .HasColumnType("TEXT"); + + b.Property("TargetConfig") + .HasColumnType("TEXT"); + + b.Property("TargetId") + .HasColumnType("TEXT"); + + b.Property("TaskType") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("NextRunAt"); + + b.HasIndex("TaskType"); + + b.ToTable("ScheduledTasks", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.AdminSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("SfpOntHintDismissedAt") + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AdminSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ApChannelChange", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApMac") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("Band") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("ChangedAtUtc") + .HasColumnType("TEXT"); + + b.Property("NewChannel") + .HasColumnType("INTEGER"); + + b.Property("NewWidthMhz") + .HasColumnType("INTEGER"); + + b.Property("PreviousChannel") + .HasColumnType("INTEGER"); + + b.Property("PreviousWidthMhz") + .HasColumnType("INTEGER"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ChangedAtUtc"); + + b.HasIndex("ApMac", "Band", "ChangedAtUtc"); + + b.ToTable("ApChannelChanges", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ApChannelOutcome", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApMac") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("Band") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("BucketDate") + .HasColumnType("TEXT"); + + b.Property("Channel") + .HasColumnType("INTEGER"); + + b.Property("InterferenceSum") + .HasColumnType("REAL"); + + b.Property("LastSampleUtc") + .HasColumnType("TEXT"); + + b.Property("SampleCount") + .HasColumnType("INTEGER"); + + b.Property("TxRetrySum") + .HasColumnType("REAL"); + + b.Property("UtilizationSum") + .HasColumnType("REAL"); + + b.Property("WidthMhz") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BucketDate"); + + b.HasIndex("ApMac", "Band", "Channel", "WidthMhz", "BucketDate") + .IsUnique(); + + b.ToTable("ApChannelOutcomes", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ApLocation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApMac") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("Floor") + .HasColumnType("INTEGER"); + + b.Property("HeightM") + .HasColumnType("REAL"); + + b.Property("Latitude") + .HasColumnType("REAL"); + + b.Property("Longitude") + .HasColumnType("REAL"); + + b.Property("MountType") + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.Property("OrientationDeg") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ApMac") + .IsUnique(); + + b.ToTable("ApLocations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ApNeighborSighting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApMac") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("Band") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("Bssid") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("Channel") + .HasColumnType("INTEGER"); + + b.Property("FirstSeenUtc") + .HasColumnType("TEXT"); + + b.Property("LastSeenUtc") + .HasColumnType("TEXT"); + + b.Property("SightingCount") + .HasColumnType("INTEGER"); + + b.Property("SignalDbm") + .HasColumnType("INTEGER"); + + b.Property("Ssid") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("WidthMhz") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("LastSeenUtc"); + + b.HasIndex("ApMac", "Band", "Bssid", "Channel") + .IsUnique(); + + b.ToTable("ApNeighborSightings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.AuditResult", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AuditDate") + .HasColumnType("TEXT"); + + b.Property("AuditVersion") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("ComplianceScore") + .HasColumnType("REAL"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("DeviceName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("FailedChecks") + .HasColumnType("INTEGER"); + + b.Property("FindingsJson") + .HasColumnType("TEXT"); + + b.Property("FirmwareVersion") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("IsScheduled") + .HasColumnType("INTEGER"); + + b.Property("Model") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("PassedChecks") + .HasColumnType("INTEGER"); + + b.Property("ReportDataJson") + .HasColumnType("TEXT"); + + b.Property("TotalChecks") + .HasColumnType("INTEGER"); + + b.Property("WarningChecks") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AuditDate"); + + b.HasIndex("DeviceId"); + + b.HasIndex("DeviceId", "AuditDate"); + + b.ToTable("AuditResults", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.Building", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CenterLatitude") + .HasColumnType("REAL"); + + b.Property("CenterLongitude") + .HasColumnType("REAL"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Buildings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ClientSignalLog", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApChannel") + .HasColumnType("INTEGER"); + + b.Property("ApClientCount") + .HasColumnType("INTEGER"); + + b.Property("ApMac") + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("ApModel") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("ApName") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("ApRadioBand") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("ApTxPower") + .HasColumnType("INTEGER"); + + b.Property("Band") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("BottleneckLinkSpeedMbps") + .HasColumnType("REAL"); + + b.Property("Channel") + .HasColumnType("INTEGER"); + + b.Property("ChannelWidth") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .HasMaxLength(45) + .HasColumnType("TEXT"); + + b.Property("ClientMac") + .IsRequired() + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("DeviceName") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("HopCount") + .HasColumnType("INTEGER"); + + b.Property("IsMlo") + .HasColumnType("INTEGER"); + + b.Property("Latitude") + .HasColumnType("REAL"); + + b.Property("LocationAccuracyMeters") + .HasColumnType("INTEGER"); + + b.Property("Longitude") + .HasColumnType("REAL"); + + b.Property("MloLinksJson") + .HasColumnType("TEXT"); + + b.Property("NoiseDbm") + .HasColumnType("INTEGER"); + + b.Property("Protocol") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("RxRateKbps") + .HasColumnType("INTEGER"); + + b.Property("SignalDbm") + .HasColumnType("INTEGER"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.Property("TraceHash") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TraceJson") + .HasColumnType("TEXT"); + + b.Property("TxRateKbps") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("TraceHash"); + + b.HasIndex("ClientMac", "Timestamp"); + + b.ToTable("ClientSignalLogs", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.CmConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("LastPolled") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("PollingIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("StatusPagePath") + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("Host"); + + b.ToTable("CmConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.CustomOidConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("DeviceMac") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("FieldName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Oid") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("Scope") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("ValueType") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("DeviceMac"); + + b.HasIndex("DeviceMac", "Oid") + .IsUnique(); + + b.ToTable("CustomOidConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.DeviceSshConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DeviceType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("Iperf3BinaryPath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Iperf3DurationSeconds") + .HasColumnType("INTEGER"); + + b.Property("Iperf3ParallelStreams") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("SshPassword") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("SshPrivateKeyPath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("SshUsername") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("StartIperf3Server") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("Host"); + + b.ToTable("DeviceSshConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.DismissedIssue", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DismissedAt") + .HasColumnType("TEXT"); + + b.Property("IssueKey") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IssueKey") + .IsUnique(); + + b.ToTable("DismissedIssues", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ExternalSpeedTestServer", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("IsDefault") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("Scheme") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("ServerId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ServerId") + .IsUnique(); + + b.ToTable("ExternalSpeedTestServers", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.FloorPlan", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("BuildingId") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("FloorMaterial") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("FloorNumber") + .HasColumnType("INTEGER"); + + b.Property("ImagePath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("NeLatitude") + .HasColumnType("REAL"); + + b.Property("NeLongitude") + .HasColumnType("REAL"); + + b.Property("Opacity") + .HasColumnType("REAL"); + + b.Property("SwLatitude") + .HasColumnType("REAL"); + + b.Property("SwLongitude") + .HasColumnType("REAL"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("WallsJson") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("BuildingId"); + + b.ToTable("FloorPlans", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.FloorPlanImage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("CropJson") + .HasColumnType("TEXT"); + + b.Property("FloorPlanId") + .HasColumnType("INTEGER"); + + b.Property("ImagePath") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Label") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("NeLatitude") + .HasColumnType("REAL"); + + b.Property("NeLongitude") + .HasColumnType("REAL"); + + b.Property("Opacity") + .HasColumnType("REAL"); + + b.Property("RotationDeg") + .HasColumnType("REAL"); + + b.Property("SortOrder") + .HasColumnType("INTEGER"); + + b.Property("SwLatitude") + .HasColumnType("REAL"); + + b.Property("SwLongitude") + .HasColumnType("REAL"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("FloorPlanId"); + + b.ToTable("FloorPlanImages", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.GatewaySshSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Host") + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("Iperf3Port") + .HasColumnType("INTEGER"); + + b.Property("LastTestResult") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("LastTestedAt") + .HasColumnType("TEXT"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("PrivateKeyPath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("TcMonitorPort") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("GatewaySshSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.InterfaceNameMap", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DeviceMac") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("Direction") + .HasColumnType("INTEGER"); + + b.Property("FriendlyName") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("IfAlias") + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("IfIndex") + .HasColumnType("INTEGER"); + + b.Property("IfName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("IsSfp") + .HasColumnType("INTEGER"); + + b.Property("IsWan") + .HasColumnType("INTEGER"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("PortNumber") + .HasColumnType("INTEGER"); + + b.Property("SpeedMbps") + .HasColumnType("INTEGER"); + + b.Property("WanName") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("DeviceMac"); + + b.HasIndex("DeviceMac", "IfName") + .IsUnique(); + + b.ToTable("InterfaceNameMaps", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.Iperf3Result", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClientMac") + .HasMaxLength(17) + .HasColumnType("TEXT"); + + b.Property("DeviceHost") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("DeviceName") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("DeviceType") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("Direction") + .HasColumnType("INTEGER"); + + b.Property("DownloadBitsPerSecond") + .HasColumnType("REAL"); + + b.Property("DownloadBytes") + .HasColumnType("INTEGER"); + + b.Property("DownloadJitterMs") + .HasColumnType("REAL"); + + b.Property("DownloadLatencyMs") + .HasColumnType("REAL"); + + b.Property("DownloadRetransmits") + .HasColumnType("INTEGER"); + + b.Property("DurationSeconds") + .HasColumnType("INTEGER"); + + b.Property("ErrorMessage") + .HasMaxLength(2000) + .HasColumnType("TEXT"); + + b.Property("ExternalServerName") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("JitterMs") + .HasColumnType("REAL"); + + b.Property("Latitude") + .HasColumnType("REAL"); + + b.Property("LocalIp") + .HasMaxLength(45) + .HasColumnType("TEXT"); + + b.Property("LocationAccuracyMeters") + .HasColumnType("INTEGER"); + + b.Property("Longitude") + .HasColumnType("REAL"); + + b.Property("Notes") + .HasMaxLength(2000) + .HasColumnType("TEXT"); + + b.Property("ParallelStreams") + .HasColumnType("INTEGER"); + + b.Property("PathAnalysisJson") + .HasColumnType("TEXT"); + + b.Property("PingMs") + .HasColumnType("REAL"); + + b.Property("RawDownloadJson") + .HasColumnType("TEXT"); + + b.Property("RawUploadJson") + .HasColumnType("TEXT"); + + b.Property("Success") + .HasColumnType("INTEGER"); + + b.Property("TestTime") + .HasColumnType("TEXT"); + + b.Property("UploadBitsPerSecond") + .HasColumnType("REAL"); + + b.Property("UploadBytes") + .HasColumnType("INTEGER"); + + b.Property("UploadJitterMs") + .HasColumnType("REAL"); + + b.Property("UploadLatencyMs") + .HasColumnType("REAL"); + + b.Property("UploadRetransmits") + .HasColumnType("INTEGER"); + + b.Property("UserAgent") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("WanName") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("WanNetworkGroup") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("WifiChannel") + .HasColumnType("INTEGER"); + + b.Property("WifiIsMlo") + .HasColumnType("INTEGER"); + + b.Property("WifiMloLinksJson") + .HasColumnType("TEXT"); + + b.Property("WifiNoiseDbm") + .HasColumnType("INTEGER"); + + b.Property("WifiRadio") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("WifiRadioProto") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("WifiRxRateKbps") + .HasColumnType("INTEGER"); + + b.Property("WifiSignalDbm") + .HasColumnType("INTEGER"); + + b.Property("WifiTxRateKbps") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("DeviceHost"); + + b.HasIndex("Direction"); + + b.HasIndex("TestTime"); + + b.HasIndex("DeviceHost", "TestTime"); + + b.ToTable("Iperf3Results", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.LicenseInfo", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("ExpirationDate") + .HasColumnType("TEXT"); + + b.Property("FeaturesJson") + .HasColumnType("TEXT"); + + b.Property("IsActive") + .HasColumnType("INTEGER"); + + b.Property("IssueDate") + .HasColumnType("TEXT"); + + b.Property("LicenseKey") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("LicenseType") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("LicensedTo") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("MaxAgents") + .HasColumnType("INTEGER"); + + b.Property("MaxDevices") + .HasColumnType("INTEGER"); + + b.Property("Organization") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ExpirationDate"); + + b.HasIndex("IsActive"); + + b.HasIndex("LicenseKey") + .IsUnique(); + + b.ToTable("Licenses", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.LicenseKeyRecord", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ActivatedAt") + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("EntitlementJson") + .HasColumnType("TEXT"); + + b.Property("IssuedAt") + .HasColumnType("TEXT"); + + b.Property("LastCheckAt") + .HasColumnType("TEXT"); + + b.Property("LastCheckError") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("LicenseKey") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("Model") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.Property("NextCheckAt") + .HasColumnType("TEXT"); + + b.Property("Org") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("PaidThrough") + .HasColumnType("TEXT"); + + b.Property("PerpetualConfirmed") + .HasColumnType("INTEGER"); + + b.Property("SiteAllowance") + .HasColumnType("INTEGER"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LicenseKey") + .IsUnique(); + + b.HasIndex("NextCheckAt"); + + b.HasIndex("Status"); + + b.ToTable("LicenseKeyRecords", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.ModemConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("LastPolled") + .HasColumnType("TEXT"); + + b.Property("ModemType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("PollingIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("PrivateKeyPath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("QmiDevice") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("Host"); + + b.ToTable("ModemConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.MonitoredSfp", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DeviceMac") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("FriendlyName") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("IsMonitoredOnt") + .HasColumnType("INTEGER"); + + b.Property("LinkSpeedMbps") + .HasColumnType("INTEGER"); + + b.Property("PortName") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("SfpPart") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("SfpVendor") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IsMonitoredOnt"); + + b.HasIndex("DeviceMac", "PortName") + .IsUnique(); + + b.ToTable("MonitoredSfps", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.MonitoringInterface", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AliasIp") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("GatewayLocalIp") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("IsManuallyDeployed") + .HasColumnType("INTEGER"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(15) + .HasColumnType("TEXT"); + + b.Property("SnatEnabled") + .HasColumnType("INTEGER"); + + b.Property("SubnetPrefix") + .HasColumnType("INTEGER"); + + b.Property("TargetIp") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("WanIfName") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("WanKey") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("WanVlanId") + .HasColumnType("INTEGER"); + + b.Property("WatchdogIntervalMinutes") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AliasIp") + .IsUnique(); + + b.HasIndex("GatewayLocalIp") + .IsUnique(); + + b.HasIndex("Name") + .IsUnique(); + + b.HasIndex("TargetIp"); + + b.ToTable("MonitoringInterfaces", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.MonitoringSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AccessTechnology") + .HasColumnType("INTEGER"); + + b.Property("AeRxPowerLowDbm") + .HasColumnType("REAL"); + + b.Property("AeTempHighC") + .HasColumnType("REAL"); + + b.Property("AeTxPowerHighDbm") + .HasColumnType("REAL"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("FastPollIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("Flex25GLatencyMigrated") + .HasColumnType("INTEGER"); + + b.Property("GatewayTempHighC") + .HasColumnType("REAL"); + + b.Property("InfluxDbBucket") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("InfluxDbLongtermBucket") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("InfluxDbOrg") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("InfluxDbReachable") + .HasColumnType("INTEGER"); + + b.Property("InfluxDbToken") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("InfluxDbUrl") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("IspHealthScoreWindowHours") + .HasColumnType("INTEGER"); + + b.Property("LastInfluxDbCheck") + .HasColumnType("TEXT"); + + b.Property("LastInfluxDbError") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("LastSnmpDetection") + .HasColumnType("TEXT"); + + b.Property("LastSnmpSuccess") + .HasColumnType("TEXT"); + + b.Property("LastUpstreamDiscoveryAt") + .HasColumnType("TEXT"); + + b.Property("MediumPollIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("PhysicalLinkSourceKey") + .HasMaxLength(120) + .HasColumnType("TEXT"); + + b.Property("PonRxPowerLowDbm") + .HasColumnType("REAL"); + + b.Property("PonTempHighC") + .HasColumnType("REAL"); + + b.Property("PonTxPowerHighDbm") + .HasColumnType("REAL"); + + b.Property("SfpTempHighGenericC") + .HasColumnType("REAL"); + + b.Property("ShowCellularTab") + .HasColumnType("INTEGER"); + + b.Property("ShowCmTab") + .HasColumnType("INTEGER"); + + b.Property("ShowOntTab") + .HasColumnType("INTEGER"); + + b.Property("ShowStarlinkTab") + .HasColumnType("INTEGER"); + + b.Property("SlowPollIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("SnmpCommunity") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("SnmpDetectionState") + .HasColumnType("INTEGER"); + + b.Property("SnmpV3AuthPassword") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("SnmpV3Username") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("SnmpVersion") + .HasColumnType("INTEGER"); + + b.Property("SwitchTempHighC") + .HasColumnType("REAL"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("UpstreamDiscoveryNeedsReview") + .HasColumnType("INTEGER"); + + b.Property("WanNeighborMac") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("WanNeighborOui") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("MonitoringSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.MonitoringTarget", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Address") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("AsnName") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("AsnNumber") + .HasColumnType("INTEGER"); + + b.Property("AutoDiscovered") + .HasColumnType("INTEGER"); + + b.Property("AutoLabel") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DeviceMac") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("DiscoveredProbeMode") + .HasColumnType("INTEGER"); + + b.Property("DiscoveryMethod") + .HasColumnType("INTEGER"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("LanFlakyHintDismissedAt") + .HasColumnType("TEXT"); + + b.Property("LastVerified") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("PingCount") + .HasColumnType("INTEGER"); + + b.Property("PollIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("ProbeMode") + .HasColumnType("INTEGER"); + + b.Property("PtrHostname") + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("TargetId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("TargetType") + .HasColumnType("INTEGER"); + + b.Property("VantagePoint") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("WanContextId") + .HasColumnType("INTEGER"); + + b.Property("WanInterface") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("TargetId") + .IsUnique(); + + b.HasIndex("TargetType"); + + b.HasIndex("WanInterface"); + + b.ToTable("MonitoringTargets", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.OntConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AttachedSfpId") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("LastPolled") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("PollingIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("PrivateKeyPath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("Host"); + + b.ToTable("OntConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.OuiVendor", b => + { + b.Property("OuiPrefix") + .HasMaxLength(8) + .HasColumnType("TEXT"); + + b.Property("LastUpdated") + .HasColumnType("TEXT"); + + b.Property("VendorName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.HasKey("OuiPrefix"); + + b.ToTable("OuiVendors", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.OutageAcknowledgement", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AcknowledgedAt") + .HasColumnType("TEXT"); + + b.Property("OutageStartUtc") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("OutageStartUtc"); + + b.ToTable("OutageAcknowledgements", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.PerfTweakSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("IsManuallyDeployed") + .HasColumnType("INTEGER"); + + b.Property("TweakId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("TweakId") + .IsUnique(); + + b.ToTable("PerfTweakSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.PlannedAp", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AntennaMode") + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Floor") + .HasColumnType("INTEGER"); + + b.Property("Latitude") + .HasColumnType("REAL"); + + b.Property("Longitude") + .HasColumnType("REAL"); + + b.Property("Model") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("MountType") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("OrientationDeg") + .HasColumnType("INTEGER"); + + b.Property("TxPower24Dbm") + .HasColumnType("INTEGER"); + + b.Property("TxPower5Dbm") + .HasColumnType("INTEGER"); + + b.Property("TxPower6Dbm") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("PlannedAps", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.Site", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("IsDefault") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Notes") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SortOrder") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Slug") + .IsUnique(); + + b.ToTable("Sites", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.SiteAgent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AgentKeyHash") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("EnrolledAt") + .HasColumnType("TEXT"); + + b.Property("EnrollmentTokenHash") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("LanIp") + .HasMaxLength(45) + .HasColumnType("TEXT"); + + b.Property("LastSeenAt") + .HasColumnType("TEXT"); + + b.Property("LastVersion") + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("SiteId") + .HasColumnType("INTEGER"); + + b.Property("TokenCreatedAt") + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AgentKeyHash"); + + b.HasIndex("EnrollmentTokenHash"); + + b.HasIndex("SiteId"); + + b.ToTable("SiteAgents", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.SiteLicenseAssignment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("LicenseKeyRecordId") + .HasColumnType("INTEGER"); + + b.Property("SiteId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("LicenseKeyRecordId"); + + b.HasIndex("SiteId") + .IsUnique(); + + b.ToTable("SiteLicenseAssignments", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.SqmBaseline", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AvgBytesIn") + .HasColumnType("INTEGER"); + + b.Property("AvgBytesOut") + .HasColumnType("INTEGER"); + + b.Property("AvgJitter") + .HasColumnType("REAL"); + + b.Property("AvgLatency") + .HasColumnType("REAL"); + + b.Property("AvgPacketLoss") + .HasColumnType("REAL"); + + b.Property("AvgUtilization") + .HasColumnType("REAL"); + + b.Property("BaselineEnd") + .HasColumnType("TEXT"); + + b.Property("BaselineHours") + .HasColumnType("INTEGER"); + + b.Property("BaselineStart") + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("HourlyDataJson") + .HasColumnType("TEXT"); + + b.Property("InterfaceId") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("InterfaceName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("MaxJitter") + .HasColumnType("REAL"); + + b.Property("MaxPacketLoss") + .HasColumnType("REAL"); + + b.Property("MedianBytesIn") + .HasColumnType("INTEGER"); + + b.Property("MedianBytesOut") + .HasColumnType("INTEGER"); + + b.Property("P95Latency") + .HasColumnType("REAL"); + + b.Property("P99Latency") + .HasColumnType("REAL"); + + b.Property("PeakBytesIn") + .HasColumnType("INTEGER"); + + b.Property("PeakBytesOut") + .HasColumnType("INTEGER"); + + b.Property("PeakLatency") + .HasColumnType("REAL"); + + b.Property("PeakUtilization") + .HasColumnType("REAL"); + + b.Property("RecommendedDownloadMbps") + .HasColumnType("REAL"); + + b.Property("RecommendedUploadMbps") + .HasColumnType("REAL"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("BaselineStart"); + + b.HasIndex("DeviceId"); + + b.HasIndex("InterfaceId"); + + b.HasIndex("DeviceId", "InterfaceId") + .IsUnique(); + + b.ToTable("SqmBaselines", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.SqmWanConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("BaselineLatencyMs") + .HasColumnType("REAL"); + + b.Property("BootDelaySeconds") + .HasColumnType("INTEGER"); + + b.Property("CongestionSeverity") + .HasColumnType("REAL"); + + b.Property("ConnectionType") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Interface") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("LatencyThresholdMs") + .HasColumnType("REAL"); + + b.Property("LinkSpeedOverrideMbps") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("NominalDownloadMbps") + .HasColumnType("INTEGER"); + + b.Property("NominalUploadMbps") + .HasColumnType("INTEGER"); + + b.Property("PingHost") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("SpeedtestEveningHour") + .HasColumnType("INTEGER"); + + b.Property("SpeedtestEveningMinute") + .HasColumnType("INTEGER"); + + b.Property("SpeedtestMorningHour") + .HasColumnType("INTEGER"); + + b.Property("SpeedtestMorningMinute") + .HasColumnType("INTEGER"); + + b.Property("SpeedtestServerId") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("WanNumber") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("WanNumber") + .IsUnique(); + + b.ToTable("SqmWanConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.StarlinkConfiguration", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("TEXT"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("LastPolled") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("PollingIntervalSeconds") + .HasColumnType("INTEGER"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("Provider") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Enabled"); + + b.HasIndex("Host"); + + b.ToTable("StarlinkConfigurations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.SystemSetting", b => + { + b.Property("Key") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasMaxLength(2000) + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("SystemSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.UniFiConnectionSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ApiKey") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("ControllerUrl") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("IgnoreControllerSSLErrors") + .HasColumnType("INTEGER"); + + b.Property("IsConfigured") + .HasColumnType("INTEGER"); + + b.Property("LastConnectedAt") + .HasColumnType("TEXT"); + + b.Property("LastError") + .HasMaxLength(1000) + .HasColumnType("TEXT"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("RememberCredentials") + .HasColumnType("INTEGER"); + + b.Property("Site") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Username") + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("UniFiConnectionSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.UniFiSshSettings", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("LastTestResult") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("LastTestedAt") + .HasColumnType("TEXT"); + + b.Property("Password") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Port") + .HasColumnType("INTEGER"); + + b.Property("PrivateKeyPath") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("Username") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("UniFiSshSettings", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.UpnpNote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("HostIp") + .IsRequired() + .HasMaxLength(45) + .HasColumnType("TEXT"); + + b.Property("Note") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Port") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("Protocol") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("HostIp", "Port", "Protocol") + .IsUnique(); + + b.ToTable("UpnpNotes", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.UpstreamDiscovery", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AncestorHopIps") + .HasMaxLength(2000) + .HasColumnType("TEXT"); + + b.Property("AsnName") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("AsnNumber") + .HasColumnType("INTEGER"); + + b.Property("HopIp") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("HopNumber") + .HasColumnType("INTEGER"); + + b.Property("IsActive") + .HasColumnType("INTEGER"); + + b.Property("LastTracerouteAt") + .HasColumnType("TEXT"); + + b.Property("LastValidated") + .HasColumnType("TEXT"); + + b.Property("MonitoringTargetId") + .HasColumnType("INTEGER"); + + b.Property("Role") + .HasColumnType("INTEGER"); + + b.Property("WanInterface") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AsnNumber"); + + b.HasIndex("IsActive"); + + b.HasIndex("MonitoringTargetId"); + + b.ToTable("UpstreamDiscoveries", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.WanContext", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AgentId") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Description") + .HasMaxLength(500) + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("ProbeSourceIp") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("WanContexts"); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.WanDataUsageConfig", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("BillingCycleDayOfMonth") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DataCapGb") + .HasColumnType("REAL"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("LastResetAt") + .HasColumnType("TEXT"); + + b.Property("ManualAdjustmentGb") + .HasColumnType("REAL"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("ResetMode") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.Property("WanKey") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.Property("WarningThresholdPercent") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("WanKey") + .IsUnique(); + + b.ToTable("WanDataUsageConfigs", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.WanDataUsageHistory", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CapGb") + .HasColumnType("REAL"); + + b.Property("CycleEnd") + .HasColumnType("TEXT"); + + b.Property("CycleStart") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("RecordedAt") + .HasColumnType("TEXT"); + + b.Property("UsedGb") + .HasColumnType("REAL"); + + b.Property("WanKey") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("WanKey", "CycleStart") + .IsUnique(); + + b.ToTable("WanDataUsageHistory", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.WanDataUsageSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("GatewayBootTime") + .HasColumnType("TEXT"); + + b.Property("IsBaseline") + .HasColumnType("INTEGER"); + + b.Property("IsCounterReset") + .HasColumnType("INTEGER"); + + b.Property("RxBytes") + .HasColumnType("INTEGER"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.Property("TxBytes") + .HasColumnType("INTEGER"); + + b.Property("WanKey") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("WanKey", "Timestamp"); + + b.ToTable("WanDataUsageSnapshots", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.WanDiscoveryContext", b => + { + b.Property("WanInterface") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("AccessTechnology") + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("L2NeighborIp") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("L2NeighborMac") + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.Property("L2NeighborOui") + .HasMaxLength(200) + .HasColumnType("TEXT"); + + b.Property("LastDiscoveryAt") + .HasColumnType("TEXT"); + + b.Property("NeedsReview") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("WanInterface"); + + b.ToTable("WanDiscoveryContexts", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.WanSteerTrafficClass", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("DstCidrsJson") + .HasColumnType("TEXT"); + + b.Property("DstPortsJson") + .HasColumnType("TEXT"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("TEXT"); + + b.Property("Probability") + .HasColumnType("REAL"); + + b.Property("Protocol") + .HasMaxLength(10) + .HasColumnType("TEXT"); + + b.Property("SortOrder") + .HasColumnType("INTEGER"); + + b.Property("SrcCidrsJson") + .HasColumnType("TEXT"); + + b.Property("SrcMacsJson") + .HasColumnType("TEXT"); + + b.Property("SrcPortsJson") + .HasColumnType("TEXT"); + + b.Property("TargetWanKey") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("SortOrder"); + + b.ToTable("WanSteerTrafficClasses", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Threats.Models.CrowdSecReputation", b => + { + b.Property("Ip") + .HasColumnType("TEXT"); + + b.Property("ExpiresAt") + .HasColumnType("TEXT"); + + b.Property("FetchedAt") + .HasColumnType("TEXT"); + + b.Property("ReputationJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Ip"); + + b.HasIndex("ExpiresAt"); + + b.ToTable("CrowdSecReputations", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Threats.Models.ThreatEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Action") + .HasColumnType("INTEGER"); + + b.Property("Asn") + .HasColumnType("INTEGER"); + + b.Property("AsnOrg") + .HasColumnType("TEXT"); + + b.Property("BytesTotal") + .HasColumnType("INTEGER"); + + b.Property("Category") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("City") + .HasColumnType("TEXT"); + + b.Property("CountryCode") + .HasColumnType("TEXT"); + + b.Property("DestIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DestPort") + .HasColumnType("INTEGER"); + + b.Property("Direction") + .HasColumnType("TEXT"); + + b.Property("Domain") + .HasColumnType("TEXT"); + + b.Property("EventSource") + .HasColumnType("INTEGER"); + + b.Property("FlowDurationMs") + .HasColumnType("INTEGER"); + + b.Property("InnerAlertId") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("KillChainStage") + .HasColumnType("INTEGER"); + + b.Property("Latitude") + .HasColumnType("REAL"); + + b.Property("Longitude") + .HasColumnType("REAL"); + + b.Property("NetworkName") + .HasColumnType("TEXT"); + + b.Property("PatternId") + .HasColumnType("INTEGER"); + + b.Property("Protocol") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("RiskLevel") + .HasColumnType("TEXT"); + + b.Property("Service") + .HasColumnType("TEXT"); + + b.Property("Severity") + .HasColumnType("INTEGER"); + + b.Property("SignatureId") + .HasColumnType("INTEGER"); + + b.Property("SignatureName") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SourceIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("SourcePort") + .HasColumnType("INTEGER"); + + b.Property("Timestamp") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("EventSource"); + + b.HasIndex("InnerAlertId") + .IsUnique(); + + b.HasIndex("KillChainStage"); + + b.HasIndex("PatternId"); + + b.HasIndex("Timestamp"); + + b.HasIndex("DestPort", "Timestamp"); + + b.HasIndex("SourceIp", "Timestamp"); + + b.ToTable("ThreatEvents", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Threats.Models.ThreatNoiseFilter", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("Description") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DestIp") + .HasColumnType("TEXT"); + + b.Property("DestPort") + .HasColumnType("INTEGER"); + + b.Property("Enabled") + .HasColumnType("INTEGER"); + + b.Property("SourceIp") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("ThreatNoiseFilters", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Threats.Models.ThreatPattern", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Confidence") + .HasColumnType("REAL"); + + b.Property("DedupKey") + .HasColumnType("TEXT"); + + b.Property("Description") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("DetectedAt") + .HasColumnType("TEXT"); + + b.Property("EventCount") + .HasColumnType("INTEGER"); + + b.Property("FirstSeen") + .HasColumnType("TEXT"); + + b.Property("LastAlertedAt") + .HasColumnType("TEXT"); + + b.Property("LastSeen") + .HasColumnType("TEXT"); + + b.Property("PatternType") + .HasColumnType("INTEGER"); + + b.Property("SourceIpsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TargetPort") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("PatternType", "DetectedAt"); + + b.ToTable("ThreatPatterns", (string)null); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.FloorPlan", b => + { + b.HasOne("NetworkOptimizer.Storage.Models.Building", "Building") + .WithMany("Floors") + .HasForeignKey("BuildingId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Building"); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.FloorPlanImage", b => + { + b.HasOne("NetworkOptimizer.Storage.Models.FloorPlan", "FloorPlan") + .WithMany("Images") + .HasForeignKey("FloorPlanId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("FloorPlan"); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.SiteLicenseAssignment", b => + { + b.HasOne("NetworkOptimizer.Storage.Models.LicenseKeyRecord", null) + .WithMany() + .HasForeignKey("LicenseKeyRecordId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("NetworkOptimizer.Storage.Models.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("NetworkOptimizer.Threats.Models.ThreatEvent", b => + { + b.HasOne("NetworkOptimizer.Threats.Models.ThreatPattern", "Pattern") + .WithMany("Events") + .HasForeignKey("PatternId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("Pattern"); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.Building", b => + { + b.Navigation("Floors"); + }); + + modelBuilder.Entity("NetworkOptimizer.Storage.Models.FloorPlan", b => + { + b.Navigation("Images"); + }); + + modelBuilder.Entity("NetworkOptimizer.Threats.Models.ThreatPattern", b => + { + b.Navigation("Events"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.cs b/src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.cs new file mode 100644 index 000000000..b742b3040 --- /dev/null +++ b/src/NetworkOptimizer.Storage/Migrations/20260719120000_AddOntAttachedSfpId.cs @@ -0,0 +1,30 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace NetworkOptimizer.Storage.Migrations; + +/// +/// Adds OntConfigurations.AttachedSfpId: when set, the ONT config supplements the +/// monitored SFP module with that MonitoredSfp.Id - polled on the gateway SFP +/// collection cycle and merged into that module's sfp measurement instead of +/// being polled standalone. Nullable - existing rows stay standalone. +/// +public partial class AddOntAttachedSfpId : Migration +{ + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "AttachedSfpId", + table: "OntConfigurations", + type: "INTEGER", + nullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "AttachedSfpId", + table: "OntConfigurations"); + } +} diff --git a/src/NetworkOptimizer.Storage/Migrations/NetworkOptimizerDbContextModelSnapshot.cs b/src/NetworkOptimizer.Storage/Migrations/NetworkOptimizerDbContextModelSnapshot.cs index 03f6ec251..f95678aa4 100644 --- a/src/NetworkOptimizer.Storage/Migrations/NetworkOptimizerDbContextModelSnapshot.cs +++ b/src/NetworkOptimizer.Storage/Migrations/NetworkOptimizerDbContextModelSnapshot.cs @@ -1947,6 +1947,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) .ValueGeneratedOnAdd() .HasColumnType("INTEGER"); + b.Property("AttachedSfpId") + .HasColumnType("INTEGER"); + b.Property("CreatedAt") .HasColumnType("TEXT"); diff --git a/src/NetworkOptimizer.Storage/Models/OntConfiguration.cs b/src/NetworkOptimizer.Storage/Models/OntConfiguration.cs index 476d52711..04a9c9e1e 100644 --- a/src/NetworkOptimizer.Storage/Models/OntConfiguration.cs +++ b/src/NetworkOptimizer.Storage/Models/OntConfiguration.cs @@ -45,6 +45,15 @@ public class OntConfiguration [MaxLength(500)] public string? PrivateKeyPath { get; set; } + /// + /// When set, this ONT supplements the monitored SFP module with that + /// MonitoredSfp.Id: it is polled on the gateway SFP collection cycle (the + /// standalone poll loop and PollingIntervalSeconds are bypassed) and its + /// PON-layer metrics merge into that module's sfp measurement. Requires a + /// provider implementing ISfpSupplementalOntProvider. Null = standalone ONT. + /// + public int? AttachedSfpId { get; set; } + /// Whether this ONT is enabled for polling public bool Enabled { get; set; } = true; diff --git a/src/NetworkOptimizer.Storage/Repositories/OntRepository.cs b/src/NetworkOptimizer.Storage/Repositories/OntRepository.cs index 3c54157ce..b72002a5c 100644 --- a/src/NetworkOptimizer.Storage/Repositories/OntRepository.cs +++ b/src/NetworkOptimizer.Storage/Repositories/OntRepository.cs @@ -84,6 +84,7 @@ public async Task SaveOntConfigurationAsync(OntConfiguration config, Cancellatio existing.Username = config.Username; existing.Password = config.Password; existing.PrivateKeyPath = config.PrivateKeyPath; + existing.AttachedSfpId = config.AttachedSfpId; existing.Enabled = config.Enabled; existing.PollingIntervalSeconds = config.PollingIntervalSeconds; existing.LastPolled = config.LastPolled; diff --git a/src/NetworkOptimizer.Storage/Services/MonitoringInfluxClient.cs b/src/NetworkOptimizer.Storage/Services/MonitoringInfluxClient.cs index c64b8a002..fc202f8e1 100644 --- a/src/NetworkOptimizer.Storage/Services/MonitoringInfluxClient.cs +++ b/src/NetworkOptimizer.Storage/Services/MonitoringInfluxClient.cs @@ -5,6 +5,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using NetworkOptimizer.Core.Enums; +using NetworkOptimizer.Core.Models; using NetworkOptimizer.Storage.Models; namespace NetworkOptimizer.Storage.Services; @@ -760,6 +761,64 @@ public Task WriteSfpAsync( return Task.CompletedTask; } + /// + /// Write supplemental PON-layer stats onto the sfp measurement, merging with the + /// DDM point CollectSfpForDevice writes for the same module and timestamp (same + /// series + timestamp = field union in InfluxDB). pon_link_status / fec_errors / + /// bip_errors reuse the ont measurement's field names and encodings so display + /// and alert logic can treat SFP-slot and standalone ONTs uniformly. + /// + public Task WriteSfpPonAsync( + string deviceMac, + string portName, + PonSupplementalStats stats, + DateTime timestamp) + { + if (!IsConfigured) return Task.CompletedTask; + var point = PointData.Measurement("sfp") + .Tag("device_mac", NormalizeMac(deviceMac)) + .Tag("port_name", portName) + .Timestamp(timestamp.ToUniversalTime(), WritePrecision.Ns); + + if (!string.IsNullOrEmpty(stats.PonLinkStatus)) point = point.Field("pon_link_status", stats.PonLinkStatus); + if (!string.IsNullOrEmpty(stats.PonLinkStatusPrev)) point = point.Field("pon_link_status_prev", stats.PonLinkStatusPrev); + if (stats.PloamElapsedMs.HasValue) point = point.Field("ploam_elapsed_ms", stats.PloamElapsedMs.Value); + if (stats.GtcDsState.HasValue) point = point.Field("gtc_ds_state", stats.GtcDsState.Value); + if (stats.OnuId.HasValue) point = point.Field("onu_id", stats.OnuId.Value); + if (stats.DsFecEnabled.HasValue) point = point.Field("ds_fec_enabled", stats.DsFecEnabled.Value); + if (stats.UsFecEnabled.HasValue) point = point.Field("us_fec_enabled", stats.UsFecEnabled.Value); + if (stats.OnuResponseTime.HasValue) point = point.Field("onu_response_time", stats.OnuResponseTime.Value); + if (stats.BipErrors.HasValue) point = point.Field("bip_errors", stats.BipErrors.Value); + if (stats.FecErrors.HasValue) point = point.Field("fec_errors", stats.FecErrors.Value); + if (stats.FecCorrectedWords.HasValue) point = point.Field("fec_corrected_words", stats.FecCorrectedWords.Value); + if (stats.HecCorrected.HasValue) point = point.Field("hec_corrected", stats.HecCorrected.Value); + if (stats.HecUncorrected.HasValue) point = point.Field("hec_uncorrected", stats.HecUncorrected.Value); + if (stats.BwmapCorrected.HasValue) point = point.Field("bwmap_corrected", stats.BwmapCorrected.Value); + if (stats.BwmapUncorrected.HasValue) point = point.Field("bwmap_uncorrected", stats.BwmapUncorrected.Value); + if (stats.GemTxFrames.HasValue) point = point.Field("gem_tx_frames", stats.GemTxFrames.Value); + if (stats.GemTxIdleFrames.HasValue) point = point.Field("gem_tx_idle_frames", stats.GemTxIdleFrames.Value); + if (stats.GemRxFrames.HasValue) point = point.Field("gem_rx_frames", stats.GemRxFrames.Value); + if (stats.GemRxDropped.HasValue) point = point.Field("gem_rx_dropped", stats.GemRxDropped.Value); + if (stats.AllocTotal.HasValue) point = point.Field("alloc_total", stats.AllocTotal.Value); + if (stats.AllocLost.HasValue) point = point.Field("alloc_lost", stats.AllocLost.Value); + if (stats.GpePonIngressDiscard.HasValue) point = point.Field("gpe_pon_ingress_discard", stats.GpePonIngressDiscard.Value); + if (stats.GpePonEgressDiscard.HasValue) point = point.Field("gpe_pon_egress_discard", stats.GpePonEgressDiscard.Value); + if (stats.GpePonLearningDiscard.HasValue) point = point.Field("gpe_pon_learning_discard", stats.GpePonLearningDiscard.Value); + if (stats.GpeLanIngressDiscard.HasValue) point = point.Field("gpe_lan_ingress_discard", stats.GpeLanIngressDiscard.Value); + if (stats.GpeLanEgressDiscard.HasValue) point = point.Field("gpe_lan_egress_discard", stats.GpeLanEgressDiscard.Value); + if (stats.GpeLanLearningDiscard.HasValue) point = point.Field("gpe_lan_learning_discard", stats.GpeLanLearningDiscard.Value); + if (stats.LanLinkStatus.HasValue) point = point.Field("lan_link_status", stats.LanLinkStatus.Value); + if (stats.LanTxFrames.HasValue) point = point.Field("lan_tx_frames", stats.LanTxFrames.Value); + if (stats.LanRxFrames.HasValue) point = point.Field("lan_rx_frames", stats.LanRxFrames.Value); + if (stats.LanTxDropEvents.HasValue) point = point.Field("lan_tx_drop_events", stats.LanTxDropEvents.Value); + if (stats.LanRxFcsErrors.HasValue) point = point.Field("lan_rx_fcs_err", stats.LanRxFcsErrors.Value); + if (stats.LanBufferOverflow.HasValue) point = point.Field("lan_buffer_overflow", stats.LanBufferOverflow.Value); + if (stats.SfpUptimeS.HasValue) point = point.Field("sfp_uptime_s", stats.SfpUptimeS.Value); + + Enqueue(point, longterm: true); + return Task.CompletedTask; + } + /// /// Write cellular modem signal metrics for time-series charting. /// Tags identify the modem; fields capture all available signal/band/carrier data. @@ -1803,6 +1862,107 @@ public async Task>> QuerySfpByModulesAsync( return results; } + /// One point of supplemental PON-layer stats on the sfp measurement (attached ONT config). + /// Counters are cumulative; callers derive per-interval deltas. + public class SfpPonPoint + { + public DateTime Time { get; set; } + public string? PonLinkStatus { get; set; } + public string? PonLinkStatusPrev { get; set; } + public long? OnuId { get; set; } + public long? DsFecEnabled { get; set; } + public long? UsFecEnabled { get; set; } + public long? OnuResponseTime { get; set; } + public long? SfpUptimeS { get; set; } + public long? BipErrors { get; set; } + public long? FecErrors { get; set; } + public long? FecCorrectedWords { get; set; } + public long? HecUncorrected { get; set; } + public long? GemTxFrames { get; set; } + public long? GemTxIdleFrames { get; set; } + public long? GemRxFrames { get; set; } + public long? GemRxDropped { get; set; } + public long? AllocLost { get; set; } + public long? LanRxFcsErrors { get; set; } + public long? LanTxDropEvents { get; set; } + public long? LanBufferOverflow { get; set; } + } + + /// + /// Supplemental PON-layer time-series for a set of (device_mac, port_name) pairs. + /// Only modules with an attached ONT config have these fields; others return no rows. + /// Aggregates with fn: last (counters are cumulative - mean would distort deltas). + /// + public async Task>> QuerySfpPonByModulesAsync( + IReadOnlyList<(string DeviceMac, string PortName)> modules, + DateTime from, + DateTime to, + TimeSpan? aggregateWindow = null, + CancellationToken ct = default) + { + if (!IsConfigured) await ReconfigureAsync(ct); + if (!IsConfigured || modules.Count == 0) return new Dictionary>(); + var window = aggregateWindow ?? PickAggregateWindow(to - from); + + var macFilter = string.Join(" or ", modules.Select(m => + $@"(r.device_mac == ""{NormalizeMac(m.DeviceMac)}"" and r.port_name == ""{SanitizeFluxString(m.PortName)}"")")); + + var fields = new[] + { + "pon_link_status", "pon_link_status_prev", "onu_id", "ds_fec_enabled", "us_fec_enabled", + "onu_response_time", "sfp_uptime_s", "bip_errors", "fec_errors", "fec_corrected_words", + "hec_uncorrected", "gem_tx_frames", "gem_tx_idle_frames", "gem_rx_frames", "gem_rx_dropped", + "alloc_lost", "lan_rx_fcs_err", "lan_tx_drop_events", "lan_buffer_overflow", + }; + var fieldFilter = string.Join(" or ", fields.Select(f => $@"r._field == ""{f}""")); + + var flux = $@" +from(bucket: ""{_longtermBucket}"") + |> range(start: {ToFluxInstant(from)}, stop: {ToFluxInstant(to)}) + |> filter(fn: (r) => r._measurement == ""sfp"") + |> filter(fn: (r) => {macFilter}) + |> filter(fn: (r) => {fieldFilter}) + |> aggregateWindow(every: {ToFluxDuration(window)}, fn: last, createEmpty: false) + |> pivot(rowKey:[""_time""], columnKey: [""_field""], valueColumn: ""_value"") +"; + var results = new Dictionary>(); + await foreach (var record in QueryFluxAsync(flux, ct)) + { + var mac = record.GetValueByKey("device_mac") as string ?? ""; + var port = record.GetValueByKey("port_name") as string ?? ""; + var key = $"{mac}:{port}"; + if (!results.TryGetValue(key, out var list)) + { + list = new List(); + results[key] = list; + } + list.Add(new SfpPonPoint + { + Time = ToUtc(record.GetTimeInDateTime() ?? DateTime.UtcNow), + PonLinkStatus = record.GetValueByKey("pon_link_status") as string, + PonLinkStatusPrev = record.GetValueByKey("pon_link_status_prev") as string, + OnuId = AsLongOrNull(record.GetValueByKey("onu_id")), + DsFecEnabled = AsLongOrNull(record.GetValueByKey("ds_fec_enabled")), + UsFecEnabled = AsLongOrNull(record.GetValueByKey("us_fec_enabled")), + OnuResponseTime = AsLongOrNull(record.GetValueByKey("onu_response_time")), + SfpUptimeS = AsLongOrNull(record.GetValueByKey("sfp_uptime_s")), + BipErrors = AsLongOrNull(record.GetValueByKey("bip_errors")), + FecErrors = AsLongOrNull(record.GetValueByKey("fec_errors")), + FecCorrectedWords = AsLongOrNull(record.GetValueByKey("fec_corrected_words")), + HecUncorrected = AsLongOrNull(record.GetValueByKey("hec_uncorrected")), + GemTxFrames = AsLongOrNull(record.GetValueByKey("gem_tx_frames")), + GemTxIdleFrames = AsLongOrNull(record.GetValueByKey("gem_tx_idle_frames")), + GemRxFrames = AsLongOrNull(record.GetValueByKey("gem_rx_frames")), + GemRxDropped = AsLongOrNull(record.GetValueByKey("gem_rx_dropped")), + AllocLost = AsLongOrNull(record.GetValueByKey("alloc_lost")), + LanRxFcsErrors = AsLongOrNull(record.GetValueByKey("lan_rx_fcs_err")), + LanTxDropEvents = AsLongOrNull(record.GetValueByKey("lan_tx_drop_events")), + LanBufferOverflow = AsLongOrNull(record.GetValueByKey("lan_buffer_overflow")), + }); + } + return results; + } + /// /// Query cellular modem signal metrics over time. Groups by modem_id + network_mode /// so LTE and NR5G are separate series (important for NSA dual-connectivity). diff --git a/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor b/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor index 656627eea..88922121d 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Monitoring.razor @@ -717,6 +717,23 @@ else

Temperature

+ +
@@ -1017,6 +1034,13 @@ else

Temperature

+ +
@@ -1915,7 +1939,7 @@ else var cmConfigs = await CmMonitorService.GetConfigsAsync(); _cmHasDevices = cmConfigs.Count > 0; - var ontConfigs = await OntMonitorService.GetConfigsAsync(); + var ontConfigs = await OntMonitorService.GetStandaloneConfigsAsync(); _ontHasDevices = ontConfigs.Count > 0; var cellConfigs = await CellularModemService.GetModemsAsync(); _cellularHasDevices = cellConfigs.Count > 0; diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index 4ea2b03be..86021888c 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -1565,11 +1565,33 @@ + + @if (_editingOnt.Provider == "netopt-custom") + { +
+
+ + + + Attached ONTs are polled with the gateway SFP cycle and their PON stats merge into + the selected module's SFP Stats series; the polling interval below is ignored. + Standalone ONTs show on ONT Stats instead. + +
+
+ } +
@@ -3956,6 +3978,7 @@ // External ONT Monitoring private List _ontConfigs = new(); + private List _attachableSfps = new(); private OntConfiguration _editingOnt = new() { Provider = "att-gateway", Port = 80, PollingIntervalSeconds = 300, Enabled = true }; private bool _ontFormVisible; private bool _savingOnt; @@ -5709,6 +5732,17 @@ try { _ontConfigs = await OntMonitorService.GetConfigsAsync(); + + // Attachment choices for the Network Optimizer Custom provider: the + // site's ONT-monitored SFP modules. + await using (var sfpDb = SiteDb.CreateForSite(SiteContext.Slug, SiteContext.IsDefault)) + { + _attachableSfps = await sfpDb.MonitoredSfps.AsNoTracking() + .Where(s => s.IsMonitoredOnt) + .OrderBy(s => s.DeviceMac).ThenBy(s => s.PortName) + .ToListAsync(); + } + if (_ontConfigs.Any(o => !string.IsNullOrEmpty(o.LastError))) _ontStatus = "Error"; else if (_ontConfigs.Any(o => o.Enabled)) @@ -5826,6 +5860,7 @@ Username = config.Username, Password = config.Password, PrivateKeyPath = config.PrivateKeyPath, + AttachedSfpId = config.AttachedSfpId, PollingIntervalSeconds = config.PollingIntervalSeconds, Enabled = config.Enabled, LastPolled = config.LastPolled, @@ -5846,12 +5881,17 @@ var provider = e.Value?.ToString() ?? "att-gateway"; _editingOnt.Provider = provider; _editingOnt.Port = GetOntProviderDefaultPort(provider); + // SFP attachment is only meaningful for providers that support it; clear a + // stale association so it can't linger invisibly after a provider switch. + if (provider != "netopt-custom") + _editingOnt.AttachedSfpId = null; } private static int GetOntProviderDefaultPort(string provider) => provider switch { "8311" => 443, "quantum-q1000k" => 443, + "netopt-custom" => 10012, _ => 80, }; @@ -5863,10 +5903,20 @@ "quantum-q1000k" => "Quantum Fiber Q1000K (HTTP)", "telekom-modem2" => "Telekom Glasfaser-Modem 2 (HTTP)", "nokia-xs010x-q" => "Nokia XS-010X-Q (HTTP)", + "netopt-custom" => "Network Optimizer Custom (HTTP JSON)", "generic-http-ont" => "Generic HTTP ONT", _ => provider, }; + private static string AttachableSfpLabel(MonitoredSfp sfp) + { + var name = !string.IsNullOrEmpty(sfp.FriendlyName) + ? sfp.FriendlyName + : $"{sfp.SfpVendor} {sfp.SfpPart}".Trim(); + if (string.IsNullOrWhiteSpace(name)) name = sfp.DeviceMac; + return $"{name} - port {sfp.PortName}"; + } + // --- Starlink Monitoring --- private async Task LoadStarlinkSettings() diff --git a/src/NetworkOptimizer.Web/Components/Shared/OntStatsPanel.razor b/src/NetworkOptimizer.Web/Components/Shared/OntStatsPanel.razor index b5275a699..3af1a35c2 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/OntStatsPanel.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/OntStatsPanel.razor @@ -224,6 +224,16 @@ else
+ var sfpPonState = PonLinkStateExtensions.ParsePonLinkState(stats?.PonLinkStatus); + // FEC disabled on the OLT profile => FEC counters stay 0; HEC is the live signal. + var fecOff = stats?.FecEnabled == false; + var midErr = fecOff ? stats?.HecUncorrected : stats?.FecErrors; + var midErrLabel = fecOff ? "HEC" : "FEC"; + var midErrTip = fecOff ? "uncorrectable HEC header errors (payload FEC disabled)" : "uncorrectable FEC codewords"; + var hasErrCounts = stats?.BipErrors.HasValue == true || midErr.HasValue || stats?.GemRxDropped.HasValue == true; + // Supplemental PON data present: move SFP Voltage to the Stats column to keep the + // two columns even (Connection gains PON Status + error counts). + var hasPonSupplement = sfpPonState != PonLinkState.Unknown || hasErrCounts;
Connection
@@ -240,6 +250,13 @@ else Link Type: @LinkTypeLabel(current)
+ @if (sfpPonState != PonLinkState.Unknown) + { +
+ PON Status: + @sfpPonState.ToDisplayString() +
+ } @if (current.LinkSpeedMbps.HasValue) {
@@ -258,12 +275,22 @@ else
} -
- SFP Voltage: - - @(stats?.VoltageV.HasValue == true ? $"{stats.VoltageV.Value:0.0#} V" : "-") - -
+ @if (hasErrCounts) + { +
+ @($"BIP/{midErrLabel}/Drops:") + @FmtCount(stats?.BipErrors) / @FmtCount(midErr) / @FmtCount(stats?.GemRxDropped) +
+ } + @if (!hasPonSupplement) + { +
+ SFP Voltage: + + @(stats?.VoltageV.HasValue == true ? $"{stats.VoltageV.Value:0.0#} V" : "-") + +
+ }
@@ -293,6 +320,15 @@ else @(stats?.TemperatureC.HasValue == true ? $"{stats.TemperatureC.Value:0.0#} °C" : "-")
+ @if (hasPonSupplement) + { +
+ SFP Voltage: + + @(stats?.VoltageV.HasValue == true ? $"{stats.VoltageV.Value:0.0#} V" : "-") + +
+ } @@ -378,7 +414,7 @@ else _fabricTargets = await db.MonitoringTargets.AsNoTracking() .Where(t => t.TargetType == MonitoringTargetType.Fabric) .ToListAsync(); - _externalOnts = await ExternalOntService.GetConfigsAsync(); + _externalOnts = await ExternalOntService.GetStandaloneConfigsAsync(); if (_currentIndex >= TotalCount) _currentIndex = 0; RefreshExternalStats(); @@ -512,6 +548,8 @@ else ? $"{mbps / 1000.0:0.##} Gbps" : $"{mbps} Mbps"; + private static string FmtCount(long? v) => v.HasValue ? v.Value.ToString("N0") : "-"; + public string? CurrentModuleKey { get diff --git a/src/NetworkOptimizer.Web/Endpoints/OntChartEndpoints.cs b/src/NetworkOptimizer.Web/Endpoints/OntChartEndpoints.cs index 6ae8457eb..edbe1177f 100644 --- a/src/NetworkOptimizer.Web/Endpoints/OntChartEndpoints.cs +++ b/src/NetworkOptimizer.Web/Endpoints/OntChartEndpoints.cs @@ -35,7 +35,9 @@ public static void Map(WebApplication app) var data = await influx.QueryOntAsync(queryFrom, queryTo, ontId, ct: ct); - var configs = await ontService.GetConfigsAsync(); + // Standalone configs only: an attached config writes to the sfp + // measurement, not ont, and must never appear as a standalone ONT series. + var configs = await ontService.GetStandaloneConfigsAsync(); var nameMap = configs.ToDictionary(c => c.Id.ToString(), c => c.Name); // Only surface ONTs that still have a config. Deleting an ONT config @@ -47,11 +49,15 @@ public static void Map(WebApplication app) { var name = nameMap[kvp.Key]; - return new + // FEC/BIP counters are cumulative; the chart wants per-interval deltas + // (CM Stats style). A negative step is a device counter reset - null + // (a gap), not a bogus spike. + var pts = kvp.Value.OrderBy(p => p.Time).ToList(); + var items = new List(pts.Count); + MonitoringInfluxClient.OntPoint? prev = null; + foreach (var p in pts) { - id = kvp.Key, - label = name, - data = kvp.Value.Select(p => new + items.Add(new { time = p.Time.ToString("o"), rx = p.RxPowerDbm, @@ -59,11 +65,24 @@ public static void Map(WebApplication app) temp = p.TemperatureC, voltage = p.VoltageV, bias = p.BiasMa, - }) + fec = Delta(p.FecErrors, prev?.FecErrors), + bip = Delta(p.BipErrors, prev?.BipErrors), + }); + prev = p; + } + + return new + { + id = kvp.Key, + label = name, + data = items, }; }); return Results.Ok(new { devices = result }); }); } + + private static long? Delta(long? cur, long? prev) => + cur is long c && prev is long p && c >= p ? c - p : null; } diff --git a/src/NetworkOptimizer.Web/Endpoints/SfpChartEndpoints.cs b/src/NetworkOptimizer.Web/Endpoints/SfpChartEndpoints.cs index 15938d67e..d61bbf596 100644 --- a/src/NetworkOptimizer.Web/Endpoints/SfpChartEndpoints.cs +++ b/src/NetworkOptimizer.Web/Endpoints/SfpChartEndpoints.cs @@ -42,6 +42,8 @@ public static void Map(WebApplication app) var modules = sfps.Select(s => (s.DeviceMac, s.PortName)).ToList(); var data = await influx.QuerySfpByModulesAsync(modules, queryFrom, queryTo, ct: ct); + // Supplemental PON-layer series (attached ONT configs); empty for modules without one. + var ponData = await influx.QuerySfpPonByModulesAsync(modules, queryFrom, queryTo, ct: ct); var targets = await db.MonitoringTargets.AsNoTracking() .Where(t => t.TargetType == MonitoringTargetType.Fabric) @@ -76,11 +78,59 @@ public static void Map(WebApplication app) tx = p.TxPowerDbm, temp = p.TemperatureC, voltage = p.VoltageV - }) + }), + pon = BuildPonSeries(ponData.TryGetValue(key, out var ponPts) ? ponPts : null) }; }); return Results.Ok(new { modules = result }); }); } + + /// + /// Project supplemental PON points into the chart payload, converting cumulative + /// counters to per-interval deltas (CM Stats style - cumulative lines are unreadable). + /// A negative step means the ONT rebooted and reset its counters; that interval's + /// delta is null (a gap) rather than a bogus spike. Null when the module has no + /// supplemental data, so the UI can hide the PON section entirely. + /// + private static List? BuildPonSeries(List? points) + { + if (points is not { Count: > 0 }) return null; + + static long? Delta(long? cur, long? prev) => + cur is long c && prev is long p && c >= p ? c - p : null; + + var ordered = points.OrderBy(p => p.Time).ToList(); + var items = new List(ordered.Count); + MonitoringInfluxClient.SfpPonPoint? prev = null; + foreach (var p in ordered) + { + items.Add(new + { + time = p.Time.ToString("o"), + state = p.PonLinkStatus, + statePrev = p.PonLinkStatusPrev, + onuId = p.OnuId, + dsFec = p.DsFecEnabled, + usFec = p.UsFecEnabled, + respTime = p.OnuResponseTime, + uptime = p.SfpUptimeS, + bip = Delta(p.BipErrors, prev?.BipErrors), + fec = Delta(p.FecErrors, prev?.FecErrors), + fecCorr = Delta(p.FecCorrectedWords, prev?.FecCorrectedWords), + hec = Delta(p.HecUncorrected, prev?.HecUncorrected), + gemTx = Delta(p.GemTxFrames, prev?.GemTxFrames), + gemTxIdle = Delta(p.GemTxIdleFrames, prev?.GemTxIdleFrames), + gemRx = Delta(p.GemRxFrames, prev?.GemRxFrames), + gemDrop = Delta(p.GemRxDropped, prev?.GemRxDropped), + allocLost = Delta(p.AllocLost, prev?.AllocLost), + lanFcs = Delta(p.LanRxFcsErrors, prev?.LanRxFcsErrors), + lanDrop = Delta(p.LanTxDropEvents, prev?.LanTxDropEvents), + lanOvfl = Delta(p.LanBufferOverflow, prev?.LanBufferOverflow), + }); + prev = p; + } + return items; + } } diff --git a/src/NetworkOptimizer.Web/Program.cs b/src/NetworkOptimizer.Web/Program.cs index ed31bdfda..65c8d3da2 100644 --- a/src/NetworkOptimizer.Web/Program.cs +++ b/src/NetworkOptimizer.Web/Program.cs @@ -356,6 +356,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddScoped(sp => sp.GetRequiredService() .GetFor(sp.GetRequiredService().Slug).Ont); diff --git a/src/NetworkOptimizer.Web/Services/AlertRuleAutoEnable.cs b/src/NetworkOptimizer.Web/Services/AlertRuleAutoEnable.cs index 9eb47e0ed..c60c68783 100644 --- a/src/NetworkOptimizer.Web/Services/AlertRuleAutoEnable.cs +++ b/src/NetworkOptimizer.Web/Services/AlertRuleAutoEnable.cs @@ -39,6 +39,36 @@ public static async Task EnableBySourceAsync(IServiceScope scope, string source, } } + /// + /// Enables specific disabled rules (by EventTypePattern) when a capability first + /// becomes available - e.g. attaching augmented PON polling to an SFP ONT unlocks the + /// BIP/HEC error alerts. Skips if any of those patterns is already enabled, so it never + /// re-enables a rule the user turned off. Complements the startup + /// path by acting immediately on the triggering save. + /// + public static async Task EnablePatternsAsync(IServiceScope scope, IReadOnlyCollection patterns, ILogger logger) + { + try + { + var db = scope.ServiceProvider.GetRequiredService(); + + var rules = (await db.AlertRules.ToListAsync()) + .Where(r => patterns.Contains(r.EventTypePattern)) + .ToList(); + if (rules.Count == 0 || rules.Any(r => r.IsEnabled)) return; + + foreach (var rule in rules) rule.IsEnabled = true; + await db.SaveChangesAsync(); + + logger.LogInformation("Auto-enabled {Count} alert rules for patterns [{Patterns}]", + rules.Count, string.Join(", ", patterns)); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to auto-enable pattern rules"); + } + } + /// /// Enables rules that were JUST seeded (their EventTypePattern is in /// ) for a source whose monitoring is already configured diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/PhysicalLinkResolver.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/PhysicalLinkResolver.cs index 324de1a73..5cc540d8b 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/PhysicalLinkResolver.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/PhysicalLinkResolver.cs @@ -139,7 +139,9 @@ private async Task> EnumerateCandidatesAsync(NetworkOptimizerDbContex private async Task> PonCandidatesAsync(NetworkOptimizerDbContext db, CancellationToken ct) { var sfps = await SfpCandidatesAsync(db, SfpCategory.Pon, PhysicalMedium.Pon, ct); - var onts = (await db.OntConfigurations.AsNoTracking().Where(o => o.Enabled).ToListAsync(ct)) + // Attached configs represent the SFP module's PON layer (handled by the SFP + // candidate above), not a standalone ONT hop - exclude them here. + var onts = (await db.OntConfigurations.AsNoTracking().Where(o => o.Enabled && o.AttachedSfpId == null).ToListAsync(ct)) .Where(o => IsFresh(o.LastPolled, o.PollingIntervalSeconds)) .Select(o => new Cand($"ont:{o.Id}", o.Name, PhysicalMedium.Pon, o.Id, null, null)); return sfps.Concat(onts).ToList(); diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/OntAlertEvaluator.cs b/src/NetworkOptimizer.Web/Services/Monitoring/OntAlertEvaluator.cs index 5a342ed95..399cd04de 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/OntAlertEvaluator.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/OntAlertEvaluator.cs @@ -18,6 +18,10 @@ public class OntAlertEvaluator private const double RxPowerHysteresisDbm = PonThresholds.PowerHysteresisDbm; private const double TempHysteresisC = PonThresholds.TempHysteresisC; private const long FecErrorDeltaThreshold = PonThresholds.PonFecErrorSpikePerPoll; + private const long BipErrorStrictThreshold = PonThresholds.PonBipErrorSpikePerPoll; + private const long BipErrorRelaxedThreshold = PonThresholds.PonBipErrorSpikeFecOnPerPoll; + private const long HecErrorDeltaThreshold = PonThresholds.PonHecErrorSpikePerPoll; + private const string DefaultSourceUrl = "/monitoring?tab=ont"; private readonly IAlertEventBus _eventBus; private readonly ILogger _logger; @@ -47,9 +51,16 @@ public async ValueTask EvaluateAsync( double? temperatureC = null, double rxPowerLowDbm = PonThresholds.PonRxPowerLowDbm, double tempHighC = PonThresholds.PonTempHighC, + long? bipErrors = null, + long? hecErrors = null, + bool? fecEnabled = null, + string? sourceUrl = null, CancellationToken ct = default) { var state = _states.GetOrAdd(ontId, _ => new OntAlertState()); + // Where a raised alert links to. Attached SFP ONTs surface on SFP Stats, standalone + // ONTs on ONT Stats; the caller passes the deep link to the triggering module. + state.SourceUrl = string.IsNullOrEmpty(sourceUrl) ? DefaultSourceUrl : sourceUrl; if (rxPowerDbm.HasValue) { @@ -58,7 +69,22 @@ public async ValueTask EvaluateAsync( await CheckPonLink(state, ontId, ontName, ponLinkStatus, ct); - if (fecErrors.HasValue) + // BIP is always-on regardless of FEC, so it's evaluated whenever reported. + if (bipErrors.HasValue) + { + await CheckBipErrors(state, ontId, ontName, bipErrors.Value, fecEnabled, ct); + } + + // The uncorrectable-codeword signal adapts to whether payload FEC is running: + // FEC codewords when it's enabled (or unknown - the standalone-ONT default), + // HEC header errors when it's explicitly disabled and FEC counters stay flat. + // Mirrors the SFP ONT card's adaptive FEC/HEC display. + if (fecEnabled == false) + { + if (hecErrors.HasValue) + await CheckHecErrors(state, ontId, ontName, hecErrors.Value, ct); + } + else if (fecErrors.HasValue) { await CheckFecErrors(state, ontId, ontName, fecErrors.Value, ct); } @@ -88,7 +114,7 @@ await _eventBus.PublishAsync(new AlertEvent DeviceName = ontName, MetricValue = rxPower, ThresholdValue = rxPowerLowDbm, - SourceUrl = "/monitoring?tab=ont", + SourceUrl = state.SourceUrl, Tags = ["ont", "rx_power"], Context = new Dictionary { @@ -127,7 +153,7 @@ await _eventBus.PublishAsync(new AlertEvent DeviceName = ontName, MetricValue = tempC, ThresholdValue = tempHighC, - SourceUrl = "/monitoring?tab=ont", + SourceUrl = state.SourceUrl, Tags = ["ont", "temperature"], Context = new Dictionary { @@ -160,7 +186,7 @@ await _eventBus.PublishAsync(new AlertEvent Title = $"{ontName} PON link down{_siteSuffix}", Message = $"ONT {ontName} PON link is down (state: {ponLinkStatus}).", DeviceName = ontName, - SourceUrl = "/monitoring?tab=ont", + SourceUrl = state.SourceUrl, Tags = ["ont", "pon_link"], Context = new Dictionary { @@ -175,41 +201,72 @@ await _eventBus.PublishAsync(new AlertEvent } } - private async ValueTask CheckFecErrors( - OntAlertState state, int ontId, string ontName, long fecErrors, CancellationToken ct) + private ValueTask CheckFecErrors( + OntAlertState state, int ontId, string ontName, long fecErrors, CancellationToken ct) => + CheckErrorSpike(ontId, ontName, fecErrors, state.PreviousFecErrors, FecErrorDeltaThreshold, + "ont.fec_errors", "FEC error", "FEC errors", "fec", "fec_delta", + v => state.PreviousFecErrors = v, state.SourceUrl, ct); + + private ValueTask CheckBipErrors( + OntAlertState state, int ontId, string ontName, long bipErrors, bool? fecEnabled, CancellationToken ct) + { + // BIP is uncorrected data loss only when payload FEC is off; with FEC on (or unknown, + // the standalone-ONT default) it counts pre-FEC line errors FEC corrects, so relax the + // threshold to avoid flagging a healthy FEC-enabled link at its normal operating point. + var threshold = fecEnabled == false ? BipErrorStrictThreshold : BipErrorRelaxedThreshold; + return CheckErrorSpike(ontId, ontName, bipErrors, state.PreviousBipErrors, threshold, + "ont.bip_errors", "BIP error", "BIP errors", "bip", "bip_delta", + v => state.PreviousBipErrors = v, state.SourceUrl, ct); + } + + private ValueTask CheckHecErrors( + OntAlertState state, int ontId, string ontName, long hecErrors, CancellationToken ct) => + CheckErrorSpike(ontId, ontName, hecErrors, state.PreviousHecErrors, HecErrorDeltaThreshold, + "ont.hec_errors", "HEC error", "HEC errors", "hec", "hec_delta", + v => state.PreviousHecErrors = v, state.SourceUrl, ct); + + /// + /// Shared per-poll error-counter spike check. Fires when the increase since the last + /// poll exceeds the threshold; a negative step (ONT counter reset) counts as zero, so + /// a reboot never fakes a spike. The first reading only establishes the baseline. + /// + private async ValueTask CheckErrorSpike( + int ontId, string ontName, long current, long? previous, long threshold, + string eventType, string spikeLabel, string countLabel, string metricTag, string deltaKey, + Action setPrevious, string sourceUrl, CancellationToken ct) { - if (state.PreviousFecErrors.HasValue) + if (previous.HasValue) { - var delta = fecErrors - state.PreviousFecErrors.Value; + var delta = current - previous.Value; if (delta < 0) delta = 0; // counter reset - if (delta > FecErrorDeltaThreshold) + if (delta > threshold) { - _logger.LogDebug("ONT {Name} FEC error spike: {Delta} errors since last poll", ontName, delta); + _logger.LogDebug("ONT {Name} {Label} spike: {Delta} since last poll", ontName, spikeLabel, delta); await _eventBus.PublishAsync(new AlertEvent { - EventType = "ont.fec_errors", + EventType = eventType, Source = "ont", Severity = AlertSeverity.Warning, - Title = $"{ontName} FEC error spike{_siteSuffix}", - Message = $"ONT {ontName} had {delta:N0} FEC errors since last poll (threshold: {FecErrorDeltaThreshold:N0}).", + Title = $"{ontName} {spikeLabel} spike{_siteSuffix}", + Message = $"ONT {ontName} had {delta:N0} {countLabel} since last poll (threshold: {threshold:N0}).", DeviceName = ontName, MetricValue = delta, - ThresholdValue = FecErrorDeltaThreshold, - SourceUrl = "/monitoring?tab=ont", - Tags = ["ont", "fec"], + ThresholdValue = threshold, + SourceUrl = sourceUrl, + Tags = ["ont", metricTag], Context = new Dictionary { ["ont_id"] = ontId.ToString(), - ["metric"] = "fec_errors", - ["fec_delta"] = delta.ToString() + ["metric"] = $"{metricTag}_errors", + [deltaKey] = delta.ToString() } }, ct); } } - state.PreviousFecErrors = fecErrors; + setPrevious(current); } private class OntAlertState @@ -218,5 +275,8 @@ private class OntAlertState public bool PonLinkDown; public bool TempBreached; public long? PreviousFecErrors; + public long? PreviousBipErrors; + public long? PreviousHecErrors; + public string SourceUrl = DefaultSourceUrl; } } diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/PonThresholds.cs b/src/NetworkOptimizer.Web/Services/Monitoring/PonThresholds.cs index 416c93a79..4f21826ec 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/PonThresholds.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/PonThresholds.cs @@ -31,9 +31,19 @@ public static class PonThresholds /// Per-poll FEC corrected-error spike threshold. Corrected errors are benign in small /// numbers (the FEC is doing its job); a sustained rate above this corroborates a marginal optic. public const long PonFecErrorSpikePerPoll = 1000; - /// Per-poll BIP (bit-interleaved-parity) error threshold. BIP errors are uncorrected bit - /// errors, so this is stricter than the FEC corrected-error threshold. - public const long PonBipErrorSpikePerPoll = 100; + /// Per-poll BIP threshold on a link running with payload FEC DISABLED, where every BIP is + /// uncorrected bit corruption. A healthy link reads 0, so this is deliberately strict - low tens of + /// uncorrected bit errors per 5-minute poll is a real fault. + public const long PonBipErrorSpikePerPoll = 25; + /// Per-poll BIP threshold on a FEC-ENABLED (or unknown) link. There BIP counts pre-FEC line + /// errors that FEC silently corrects, so a link at the normal ITU operating point reads hundreds/poll + /// while data is perfect; only a gross rate (matching the FEC codeword line) is worth flagging. + public const long PonBipErrorSpikeFecOnPerPoll = 1000; + /// Per-poll uncorrectable-HEC (header error control) threshold. HEC guards the GEM/GTC + /// framing headers and is always active regardless of payload FEC, so it's the live error signal on a + /// FEC-disabled link. Uncorrectable header errors drop frames, and a healthy link reads ~0, so this is + /// strict. + public const long PonHecErrorSpikePerPoll = 10; // --- ONT error-count health for ISP Health (absolute count over the window, per-day). --- // A healthy PON link has a negligible BIP / uncorrectable-FEC count: 0 is ideal, a few per day diff --git a/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs b/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs index 8bbb25756..8dd371be5 100644 --- a/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs +++ b/src/NetworkOptimizer.Web/Services/MonitoringCollectionAgent.cs @@ -2,13 +2,16 @@ using System.Net; using Microsoft.EntityFrameworkCore; using NetworkOptimizer.Core.Enums; +using NetworkOptimizer.Core.Models; using NetworkOptimizer.Monitoring; using NetworkOptimizer.Monitoring.Models; using NetworkOptimizer.Monitoring.Probes; +using NetworkOptimizer.Monitoring.Providers; using NetworkOptimizer.Storage.Models; using NetworkOptimizer.Storage.Services; using NetworkOptimizer.UniFi; using NetworkOptimizer.UniFi.Models; +using NetworkOptimizer.Web.Services.OntProviders; namespace NetworkOptimizer.Web.Services; @@ -50,6 +53,9 @@ public class MonitoringCollectionAgent : BackgroundService private readonly NetworkOptimizer.Web.Services.Monitoring.MonitoringAlertEvaluator _alertEvaluator; private readonly NetworkOptimizer.Web.Services.Monitoring.SfpAlertEvaluator _sfpAlertEvaluator; private readonly NetworkOptimizer.Web.Services.Monitoring.DeviceHealthAlertEvaluator _deviceHealthAlertEvaluator; + private readonly NetworkOptimizer.Web.Services.Monitoring.OntAlertEvaluator _ontAlertEvaluator; + private readonly Dictionary _supplementalOntProviders; + private readonly SiteTunnelRouting _tunnelRouting; private readonly ILoggerFactory _loggerFactory; private readonly ILogger _logger; private readonly Licensing.LicenseStateService _licenseState; @@ -125,6 +131,8 @@ public MonitoringCollectionAgent( LocalProbeExecutor localProbe, MonitoringAlertRegistry alertRegistry, Licensing.LicenseStateService licenseState, + IEnumerable ontProviders, + SiteTunnelRouting tunnelRouting, ILoggerFactory loggerFactory, ILogger logger, string siteSlug = SiteManagementService.DefaultSiteSlug) @@ -144,6 +152,11 @@ public MonitoringCollectionAgent( _alertEvaluator = evaluators.Targets; _sfpAlertEvaluator = evaluators.Sfp; _deviceHealthAlertEvaluator = evaluators.DeviceHealth; + _ontAlertEvaluator = evaluators.Ont; + _supplementalOntProviders = ontProviders + .OfType() + .ToDictionary(p => p.ProviderKey, StringComparer.OrdinalIgnoreCase); + _tunnelRouting = tunnelRouting; _loggerFactory = loggerFactory; _logger = logger; } @@ -889,12 +902,17 @@ private async Task SlowTierCollectAsync(MonitoringSettings settings, Cancellatio var existingSfps = await db.MonitoredSfps.ToDictionaryAsync( s => (s.DeviceMac, s.PortName), s => s, ct); + // Supplemental PON stats: poll ONT configs attached to a monitored SFP module + // (Network Optimizer Custom endpoints) so their PON-layer metrics can merge into + // the sfp measurement alongside the DDM values below. + var ponSupplemental = await CollectPonSupplementalAsync(db, existingSfps, settings, ct); + // SFP collection (spec 5.9): write DDM values to the sfp measurement for every port // where UniFi reports sfp_found, and reconcile the MonitoredSfps relational row. var nowSfp = DateTime.UtcNow; foreach (var device in devices) { - CollectSfpForDevice(device, db, existingSfps, nowSfp); + CollectSfpForDevice(device, db, existingSfps, ponSupplemental, nowSfp); } // SFP threshold evaluation: check DDM values against alert thresholds @@ -2064,10 +2082,125 @@ private async Task SeedSfpLiveCacheAsync(CancellationToken ct) _logger.LogDebug("Seeded {Count} SFP live entries from InfluxDB (site {Site})", seeded, _siteSlug); } + /// + /// Poll every enabled ONT configuration attached to a monitored SFP module and + /// return the supplemental PON stats keyed by that module's (DeviceMac, PortName), + /// for merging into the sfp measurement. Attached configs bypass the standalone + /// OntMonitorService loop, so their LastPolled/LastError status is stamped here + /// (persisted by the slow tier's SaveChanges) and ONT alerts (PON link state, FEC) + /// are evaluated here too - DDM alerts stay with the SFP evaluator. + /// + private async Task> CollectPonSupplementalAsync( + NetworkOptimizerDbContext db, + Dictionary<(string DeviceMac, string PortName), MonitoredSfp> sfps, + MonitoringSettings settings, + CancellationToken ct) + { + var result = new Dictionary<(string, string), PonSupplementalStats>(); + List attached; + try + { + attached = await db.OntConfigurations + .Where(o => o.Enabled && o.AttachedSfpId != null) + .ToListAsync(ct); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Attached ONT config load failed (site {Site})", _siteSlug); + return result; + } + if (attached.Count == 0) return result; + + var sfpById = sfps.Values.Where(s => s.Id > 0).ToDictionary(s => s.Id); + var thresholds = NetworkOptimizer.Web.Services.Monitoring.SfpDdmThresholds.FromSettings(settings); + + foreach (var config in attached) + { + if (!sfpById.TryGetValue(config.AttachedSfpId!.Value, out var sfp)) + { + config.LastError = "Attached SFP module no longer exists"; + config.UpdatedAt = DateTime.UtcNow; + continue; + } + if (!_supplementalOntProviders.TryGetValue(config.Provider, out var provider)) + { + config.LastError = $"Provider '{config.Provider}' does not support SFP module attachment"; + config.UpdatedAt = DateTime.UtcNow; + continue; + } + + try + { + // Same tunnel-proxy routing the standalone ONT poll path uses, so + // attached endpoints on agent sites are reached through the relay. + var (host, port) = await _tunnelRouting.RouteAsync(_siteSlug, config.Host, config.Port); + var context = new OntPollContext + { + Id = config.Id, + Name = config.Name, + Host = host, + ConfiguredHost = config.Host, + Port = port, + }; + + var stats = await provider.PollSupplementalAsync(context, ct); + if (stats != null) + { + result[(sfp.DeviceMac, sfp.PortName)] = stats; + config.LastPolled = DateTime.UtcNow; + config.LastError = null; + + // Alert evaluation runs in its own try/catch so an alert failure is + // logged but never marks the (successful) poll as errored - matching + // the SFP/device-health evaluator call sites. FEC is only meaningful + // when the OLT profile enables it; when disabled the evaluator falls + // back to HEC, so pass the enable state. + bool? fecEnabled = stats.DsFecEnabled.HasValue || stats.UsFecEnabled.HasValue + ? stats.DsFecEnabled == 1 || stats.UsFecEnabled == 1 + : null; + // Attached ONTs surface on SFP Stats, so link the alert to that module. + var sfpUrl = $"/monitoring?tab=sfp&sfp={sfp.DeviceMac.Replace("-", ":").ToLowerInvariant()}:{sfp.PortName}"; + try + { + await _ontAlertEvaluator.EvaluateAsync( + config.Id, config.Name, + rxPowerDbm: null, + NetOptCustomPonOntProvider.ToPonLinkState(stats.PloamStateRaw), + stats.FecErrors, + temperatureC: null, + thresholds.PonRxPowerLowDbm, thresholds.PonTempHighC, + bipErrors: stats.BipErrors, + hecErrors: stats.HecUncorrected, + fecEnabled: fecEnabled, + sourceUrl: sfpUrl, + ct: ct); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "PON alert evaluation failed for {Name}", config.Name); + } + } + else + { + config.LastError = "Poll returned no data"; + } + config.UpdatedAt = DateTime.UtcNow; + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Supplemental PON poll failed for {Name} (site {Site})", config.Name, _siteSlug); + config.LastError = ex.Message; + config.UpdatedAt = DateTime.UtcNow; + } + } + return result; + } + private void CollectSfpForDevice( UniFiDeviceResponse device, NetworkOptimizerDbContext db, Dictionary<(string DeviceMac, string PortName), MonitoredSfp> existing, + IReadOnlyDictionary<(string DeviceMac, string PortName), PonSupplementalStats> ponSupplemental, DateTime timestamp) { if (device.PortTable == null || device.PortTable.Count == 0) return; @@ -2097,6 +2230,21 @@ private void CollectSfpForDevice( sfpLinkSpeedMbps: port.Speed > 0 ? port.Speed : null, timestamp: timestamp); + // Supplemental PON-layer stats from an attached ONT config merge onto the + // same series and timestamp (field union with the DDM point above). + if (ponSupplemental.TryGetValue((mac, portName), out var ponStats)) + { + _ = _influx.WriteSfpPonAsync(mac, portName, ponStats, timestamp); + // Mirror the key PON values into live stats so the SFP ONT card can show + // PON status and absolute error counters without a DB roundtrip. + bool? fecEnabled = ponStats.DsFecEnabled.HasValue || ponStats.UsFecEnabled.HasValue + ? ponStats.DsFecEnabled == 1 || ponStats.UsFecEnabled == 1 + : null; + _liveStats.RecordSfpPon(mac, portName, ponStats.PonLinkStatus, + ponStats.BipErrors, ponStats.FecErrors, ponStats.HecUncorrected, fecEnabled, + ponStats.GemRxDropped, timestamp); + } + // Mirror the values into the live-stats cache so the dashboard SFP card can // render without a DB roundtrip on every refresh. _liveStats.RecordSfp( diff --git a/src/NetworkOptimizer.Web/Services/MonitoringLiveStats.cs b/src/NetworkOptimizer.Web/Services/MonitoringLiveStats.cs index 1926928f9..28ba97975 100644 --- a/src/NetworkOptimizer.Web/Services/MonitoringLiveStats.cs +++ b/src/NetworkOptimizer.Web/Services/MonitoringLiveStats.cs @@ -469,8 +469,9 @@ public void RecordSfp(string deviceMac, string portName, double? rxDbm, double? }, // Merge: each field keeps the new value when present, otherwise preserves // the prior value. One null reading on a single sensor (e.g. bias) no - // longer wipes the others. - (_, prior) => new SfpLiveStats + // longer wipes the others. PON supplement fields (set by RecordSfpPon) are + // always carried through - this DDM path never touches them. + (_, prior) => prior with { RxPowerDbm = rxDbm ?? prior.RxPowerDbm, TxPowerDbm = txDbm ?? prior.TxPowerDbm, @@ -481,6 +482,45 @@ public void RecordSfp(string deviceMac, string portName, double? rxDbm, double? }); } + /// + /// Record the latest PON-layer supplement (from an attached ONT provider) onto the + /// module's live SFP entry, preserving the DDM readings. Absolute counters; the card + /// shows them as-is. Skips the write when nothing usable came back so a failed + /// supplement poll doesn't blank the prior good values. + /// + public void RecordSfpPon(string deviceMac, string portName, string? ponLinkStatus, + long? bipErrors, long? fecErrors, long? hecUncorrected, bool? fecEnabled, + long? gemRxDropped, DateTime timestamp) + { + if (string.IsNullOrEmpty(deviceMac) || string.IsNullOrEmpty(portName)) return; + if (string.IsNullOrEmpty(ponLinkStatus) && !bipErrors.HasValue && !fecErrors.HasValue + && !hecUncorrected.HasValue && !gemRxDropped.HasValue) + return; + + var key = (Normalize(deviceMac), portName); + _sfpStats.AddOrUpdate( + key, + _ => new SfpLiveStats + { + PonLinkStatus = ponLinkStatus, + BipErrors = bipErrors, + FecErrors = fecErrors, + HecUncorrected = hecUncorrected, + FecEnabled = fecEnabled, + GemRxDropped = gemRxDropped, + LastUpdate = timestamp + }, + (_, prior) => prior with + { + PonLinkStatus = ponLinkStatus ?? prior.PonLinkStatus, + BipErrors = bipErrors ?? prior.BipErrors, + FecErrors = fecErrors ?? prior.FecErrors, + HecUncorrected = hecUncorrected ?? prior.HecUncorrected, + FecEnabled = fecEnabled ?? prior.FecEnabled, + GemRxDropped = gemRxDropped ?? prior.GemRxDropped, + }); + } + // ---- WiFi clients (spec 5.2 client data collection) ---- /// @@ -677,6 +717,22 @@ public record SfpLiveStats public double? TemperatureC { get; init; } public double? VoltageV { get; init; } public DateTime LastUpdate { get; init; } + + /// PON activation state, influx-encoded (e.g. "operation"), from an attached + /// supplemental ONT provider. Null when the module has no PON supplement. + public string? PonLinkStatus { get; init; } + /// Absolute (cumulative) BIP error count from the PON supplement. + public long? BipErrors { get; init; } + /// Absolute (cumulative) uncorrectable FEC codewords from the PON supplement. + public long? FecErrors { get; init; } + /// Absolute (cumulative) uncorrectable HEC header errors - the always-on + /// framing-layer error signal, meaningful even when payload FEC is disabled. + public long? HecUncorrected { get; init; } + /// Whether payload FEC is enabled per the OLT profile. Null = unknown. When + /// false, the FEC counters stay 0 and HEC is the live error signal to show instead. + public bool? FecEnabled { get; init; } + /// Absolute (cumulative) dropped GEM frames from the PON supplement. + public long? GemRxDropped { get; init; } } public record PortLiveRate diff --git a/src/NetworkOptimizer.Web/Services/OntMonitorService.cs b/src/NetworkOptimizer.Web/Services/OntMonitorService.cs index 83e795754..08ae73d33 100644 --- a/src/NetworkOptimizer.Web/Services/OntMonitorService.cs +++ b/src/NetworkOptimizer.Web/Services/OntMonitorService.cs @@ -95,7 +95,8 @@ public IReadOnlyDictionary GetAllCachedStats() } /// - /// Get all ONT configurations (for UI and chart endpoints). + /// Get all ONT configurations, including those attached to an SFP module + /// (for the Settings management list). /// public async Task> GetConfigsAsync() { @@ -104,6 +105,18 @@ public async Task> GetConfigsAsync() return await repository.GetOntConfigurationsAsync(); } + /// + /// Standalone ONT configurations only - excludes configs attached to an SFP + /// module, which surface as PON data on that module (SFP Stats), not as their + /// own ONT device. Drives the ONT Stats tab, the Dashboard ONT card, and the + /// ONT chart endpoint so an attached config never appears as a standalone ONT. + /// + public async Task> GetStandaloneConfigsAsync() + { + var configs = await GetConfigsAsync(); + return configs.Where(c => c.AttachedSfpId == null).ToList(); + } + /// /// Manually poll a single ONT by ID (used by UI refresh button). /// @@ -152,8 +165,17 @@ public async Task SaveOntAsync(OntConfiguration config) if (isNew) await AlertRuleAutoEnable.EnableBySourceAsync(scope, "ont", _logger); + + // Attaching augmented PON polling to an SFP ONT unlocks the BIP/HEC error alerts - + // enable them immediately (any save that lands an attachment, new or edited), so an + // existing-ONT user doesn't have to wait for the next startup's freshly-seeded pass. + if (config.AttachedSfpId.HasValue) + await AlertRuleAutoEnable.EnablePatternsAsync(scope, AugmentedOntAlertPatterns, _logger); } + /// PON error alerts unlocked by augmented (SFP-attached) ONT polling. + private static readonly string[] AugmentedOntAlertPatterns = { "ont.bip_errors", "ont.hec_errors" }; + /// /// Delete an ONT configuration and remove cached stats. /// @@ -209,6 +231,12 @@ private async Task PollAllAsync() foreach (var config in configs) { + // Configs attached to a monitored SFP module are polled by that + // site's gateway SFP collection cycle (MonitoringCollectionAgent), + // which merges their PON stats into the module's sfp measurement. + if (config.AttachedSfpId.HasValue) + continue; + if (!forceAll && config.LastPolled.HasValue) { var elapsed = DateTime.UtcNow - config.LastPolled.Value; @@ -260,10 +288,14 @@ private async Task PollAllAsync() // Fire-and-forget write to InfluxDB WriteToInflux(config, stats); + // Standalone ONT providers report BIP where available but not HEC or the + // FEC-enable state, so FEC stays the codeword-error signal (fecEnabled null). _ = _alertEvaluator.EvaluateAsync( config.Id, config.Name, stats.RxPowerDbm, stats.PonLinkStatus, stats.FecErrors, - stats.TemperatureC, thresholds.PonRxPowerLowDbm, thresholds.PonTempHighC); + stats.TemperatureC, thresholds.PonRxPowerLowDbm, thresholds.PonTempHighC, + bipErrors: stats.BipErrors, + sourceUrl: $"/monitoring?tab=ont&ont={config.Id}"); _logger.LogDebug("ONT {Name} polled successfully: Rx={Rx} dBm", config.Name, stats.RxPowerDbm); return stats; diff --git a/src/NetworkOptimizer.Web/Services/OntProviders/NetOptCustomPonOntProvider.cs b/src/NetworkOptimizer.Web/Services/OntProviders/NetOptCustomPonOntProvider.cs new file mode 100644 index 000000000..6c3a65e25 --- /dev/null +++ b/src/NetworkOptimizer.Web/Services/OntProviders/NetOptCustomPonOntProvider.cs @@ -0,0 +1,419 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using NetworkOptimizer.Core.Models; +using NetworkOptimizer.Monitoring.Models; +using NetworkOptimizer.Monitoring.Providers; + +namespace NetworkOptimizer.Web.Services.OntProviders; + +/// +/// Provider for the "Network Optimizer Custom" PON stats JSON contract +/// (docs/features/netopt-custom-pon-contract.md): a plain HTTP endpoint, typically +/// served from the gateway or the ONT itself, returning ITU-T PON-layer stats +/// (PLOAM state, GTC status/counters, GEM/allocation counters, bridge-port +/// discards, host-link counters) as JSON. Anyone can implement the contract for +/// their ONT hardware; the first implementation gathers Lantiq `onu` CLI output +/// from a GPON SFP stick. +/// +/// Usable standalone like any ONT provider (PON state and error counters flow to +/// the ont measurement), but designed to be attached to a monitored SFP module +/// (ISfpSupplementalOntProvider), where its metrics merge into that module's sfp +/// measurement on the gateway SFP poll cycle. +/// +public class NetOptCustomPonOntProvider : ISfpSupplementalOntProvider +{ + private readonly ILogger _logger; + + /// + /// Reference endpoints are one-shot listeners (a netcat accept loop) that serve a + /// single connection at a time and re-gather stats per request, so requests to the + /// same endpoint must never overlap. Gated per (host, port) so distinct endpoints - + /// different devices, different sites - still poll in parallel (this is a shared + /// singleton across all sites). + /// + private readonly ConcurrentDictionary _requestGates = new(); + + private const int DefaultPort = 10012; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + NumberHandling = JsonNumberHandling.AllowReadingFromString, + }; + + public NetOptCustomPonOntProvider(ILogger logger) + { + _logger = logger; + } + + public string ProviderKey => "netopt-custom"; + public string DisplayName => "Network Optimizer Custom (HTTP JSON)"; + + public async Task PollAsync(OntPollContext context, CancellationToken cancellationToken = default) + { + var payload = await FetchPayloadAsync(context, cancellationToken); + if (payload == null) return null; + var stats = MapToOntStats(payload); + stats.DeviceHost = context.ConfiguredHost ?? context.Host; + stats.DeviceName = context.Name; + return stats; + } + + public async Task PollSupplementalAsync( + OntPollContext context, CancellationToken cancellationToken = default) + { + var payload = await FetchPayloadAsync(context, cancellationToken); + return payload == null ? null : MapToSupplemental(payload); + } + + public async Task<(bool Success, string Message)> TestConnectionAsync( + OntPollContext context, CancellationToken cancellationToken = default) + { + try + { + var payload = await FetchPayloadAsync(context, cancellationToken, throwOnError: true); + if (payload == null) + return (false, "Endpoint returned no parseable stats"); + + var state = ToPonLinkState(payload.Ploam?.CurrState); + var detail = state != PonLinkState.Unknown + ? $"PLOAM {state.ToDisplayString()}" + : "no PLOAM section"; + if (payload.GtcStatus?.OnuId != null) + detail += $", ONU ID {payload.GtcStatus.OnuId}"; + if (payload.SfpUptimeS != null) + { + var up = TimeSpan.FromSeconds(payload.SfpUptimeS.Value); + detail += $", up {(int)up.TotalDays}d {up.Hours}h {up.Minutes}m"; + } + return (true, $"Connected - {detail}"); + } + catch (HttpRequestException ex) + { + return (false, $"Connection failed: {ex.Message}"); + } + catch (TaskCanceledException) + { + return (false, "Connection timed out"); + } + catch (Exception ex) + { + return (false, $"Unexpected error: {ex.Message}"); + } + } + + /// + /// Fetch and parse the endpoint. Returns null on failure (logged at Debug) + /// unless (the Test button wants the message). + /// + private async Task FetchPayloadAsync( + OntPollContext context, CancellationToken cancellationToken, bool throwOnError = false) + { + var port = context.Port > 0 ? context.Port : DefaultPort; + var url = $"http://{context.Host}:{port}/"; + + var gate = _requestGates.GetOrAdd($"{context.Host}:{port}", _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken); + try + { + // Generous timeout: reference implementations gather stats on demand + // (e.g. an SSH round-trip into the SFP stick) before responding. + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(15) }; + var json = await client.GetStringAsync(url, cancellationToken); + + var payload = ParsePayload(json); + if (payload == null) + { + _logger.LogDebug("PON stats endpoint {Host} returned unparseable payload", context.ConfiguredHost ?? context.Host); + if (throwOnError) throw new InvalidOperationException("Response was not valid PON stats JSON"); + return null; + } + + if (!string.IsNullOrEmpty(payload.Error)) + { + _logger.LogDebug("PON stats endpoint {Host} reported error: {Error} - {Message}", + context.ConfiguredHost ?? context.Host, payload.Error, payload.Message); + if (throwOnError) + throw new InvalidOperationException( + string.IsNullOrEmpty(payload.Message) ? payload.Error : $"{payload.Error}: {payload.Message}"); + return null; + } + + return payload; + } + catch (Exception ex) when (!throwOnError) + { + _logger.LogDebug(ex, "PON stats poll failed for {Host}", context.ConfiguredHost ?? context.Host); + return null; + } + finally + { + gate.Release(); + } + } + + internal static NetOptCustomPonPayload? ParsePayload(string json) + { + try + { + return JsonSerializer.Deserialize(json, JsonOptions); + } + catch (JsonException) + { + return null; + } + } + + /// + /// Map the contract payload onto the shared PON supplemental DTO. Standard + /// concepts keep their standard encodings: PLOAM states use the same strings + /// as the ont measurement's pon_link_status; fec_errors is uncorrectable + /// codewords; bip_errors is the raw BIP counter. + /// + internal static PonSupplementalStats MapToSupplemental(NetOptCustomPonPayload p) => new() + { + PloamStateRaw = p.Ploam?.CurrState, + PonLinkStatus = EncodePloamState(p.Ploam?.CurrState), + PonLinkStatusPrev = EncodePloamState(p.Ploam?.PreviousState), + PloamElapsedMs = p.Ploam?.ElapsedMsec, + GtcDsState = p.GtcStatus?.DsState, + OnuId = p.GtcStatus?.OnuId, + DsFecEnabled = p.GtcStatus?.DsFecEnable, + UsFecEnabled = p.GtcStatus?.UsFecEnable, + OnuResponseTime = p.GtcStatus?.OnuResponseTime, + BipErrors = p.GtcCounters?.Bip, + FecErrors = p.GtcCounters?.FecWordsUncorr, + FecCorrectedWords = p.GtcCounters?.FecWordsCorr, + HecCorrected = p.GtcCounters?.HecErrorCorr, + HecUncorrected = p.GtcCounters?.HecErrorUncorr, + BwmapCorrected = p.GtcCounters?.BwmapErrorCorr, + BwmapUncorrected = p.GtcCounters?.BwmapErrorUncorr, + GemTxFrames = p.GtcCounters?.TxGemFramesTotal, + GemTxIdleFrames = p.GtcCounters?.TxGemIdleFramesTotal, + GemRxFrames = p.GtcCounters?.RxGemFramesTotal, + GemRxDropped = p.GtcCounters?.RxGemFramesDropped, + AllocTotal = p.GtcCounters?.AllocationsTotal, + AllocLost = p.GtcCounters?.AllocationsLost, + GpePonIngressDiscard = p.GpePon?.IbpDiscard, + GpePonEgressDiscard = p.GpePon?.EbpDiscard, + GpePonLearningDiscard = p.GpePon?.LearningDiscard, + GpeLanIngressDiscard = p.GpeLan?.IbpDiscard, + GpeLanEgressDiscard = p.GpeLan?.EbpDiscard, + GpeLanLearningDiscard = p.GpeLan?.LearningDiscard, + LanLinkStatus = p.Lan?.LinkStatus, + LanTxFrames = p.LanCounters?.TxFrames, + LanRxFrames = p.LanCounters?.RxFrames, + LanTxDropEvents = p.LanCounters?.TxDropEvents, + LanRxFcsErrors = p.LanCounters?.RxFcsErr, + LanBufferOverflow = p.LanCounters?.BufferOverflow, + SfpUptimeS = p.SfpUptimeS, + }; + + /// + /// Standalone mapping: the standard OntStats fields this contract can serve. + /// DDM optics readings are absent by design - in the SFP-module scenario the + /// gateway's DDM poll owns those. + /// + internal static OntStats MapToOntStats(NetOptCustomPonPayload p) => new() + { + Timestamp = DateTime.UtcNow, + PonLinkStatus = ToPonLinkState(p.Ploam?.CurrState), + FecErrors = p.GtcCounters?.FecWordsUncorr, + BipErrors = p.GtcCounters?.Bip, + }; + + /// Raw PLOAM state number (1-7 = O1-O7) to the shared enum. + internal static PonLinkState ToPonLinkState(long? state) => + state is >= 1 and <= 7 ? (PonLinkState)(int)state.Value : PonLinkState.Unknown; + + private static string? EncodePloamState(long? state) => + state == null ? null : ToPonLinkState(state).ToInfluxValue(); +} + +/// +/// The "Network Optimizer Custom" PON stats JSON contract, v1. Every section is +/// optional - implementations serve what their hardware exposes, and absent +/// sections are simply not recorded. Counters are cumulative since ONT boot. +/// See docs/features/netopt-custom-pon-contract.md for field semantics. +/// +public class NetOptCustomPonPayload +{ + /// Machine-readable failure code (e.g. "sfp_unreachable"). Non-null means the poll failed. + [JsonPropertyName("error")] + public string? Error { get; set; } + + /// Human-readable failure detail accompanying . + [JsonPropertyName("message")] + public string? Message { get; set; } + + [JsonPropertyName("lan")] + public LanSection? Lan { get; set; } + + [JsonPropertyName("lan_counters")] + public LanCountersSection? LanCounters { get; set; } + + [JsonPropertyName("ploam")] + public PloamSection? Ploam { get; set; } + + [JsonPropertyName("gtc_status")] + public GtcStatusSection? GtcStatus { get; set; } + + [JsonPropertyName("gtc_counters")] + public GtcCountersSection? GtcCounters { get; set; } + + [JsonPropertyName("gpe_pon")] + public GpeBridgePortSection? GpePon { get; set; } + + [JsonPropertyName("gpe_lan")] + public GpeBridgePortSection? GpeLan { get; set; } + + /// Seconds since the ONT module booted. + [JsonPropertyName("sfp_uptime_s")] + public long? SfpUptimeS { get; set; } + + public class LanSection + { + [JsonPropertyName("mode")] + public long? Mode { get; set; } + + [JsonPropertyName("link_status")] + public long? LinkStatus { get; set; } + + [JsonPropertyName("phy_duplex")] + public long? PhyDuplex { get; set; } + } + + public class LanCountersSection + { + [JsonPropertyName("tx_frames")] + public long? TxFrames { get; set; } + + [JsonPropertyName("rx_frames")] + public long? RxFrames { get; set; } + + [JsonPropertyName("tx_drop_events")] + public long? TxDropEvents { get; set; } + + [JsonPropertyName("rx_fcs_err")] + public long? RxFcsErr { get; set; } + + [JsonPropertyName("buffer_overflow")] + public long? BufferOverflow { get; set; } + } + + public class PloamSection + { + /// Current ITU-T activation state, numeric: 1-7 = O1-O7. + [JsonPropertyName("curr_state")] + public long? CurrState { get; set; } + + [JsonPropertyName("previous_state")] + public long? PreviousState { get; set; } + + [JsonPropertyName("elapsed_msec")] + public long? ElapsedMsec { get; set; } + } + + public class GtcStatusSection + { + [JsonPropertyName("ds_state")] + public long? DsState { get; set; } + + [JsonPropertyName("onu_id")] + public long? OnuId { get; set; } + + [JsonPropertyName("ds_fec_enable")] + public long? DsFecEnable { get; set; } + + [JsonPropertyName("us_fec_enable")] + public long? UsFecEnable { get; set; } + + [JsonPropertyName("onu_response_time")] + public long? OnuResponseTime { get; set; } + } + + public class GtcCountersSection + { + [JsonPropertyName("bip")] + public long? Bip { get; set; } + + [JsonPropertyName("hec_error_corr")] + public long? HecErrorCorr { get; set; } + + [JsonPropertyName("hec_error_uncorr")] + public long? HecErrorUncorr { get; set; } + + [JsonPropertyName("bwmap_error_corr")] + public long? BwmapErrorCorr { get; set; } + + [JsonPropertyName("bwmap_error_uncorr")] + public long? BwmapErrorUncorr { get; set; } + + [JsonPropertyName("fec_error_corr")] + public long? FecErrorCorr { get; set; } + + [JsonPropertyName("fec_words_corr")] + public long? FecWordsCorr { get; set; } + + [JsonPropertyName("fec_words_uncorr")] + public long? FecWordsUncorr { get; set; } + + [JsonPropertyName("fec_words_total")] + public long? FecWordsTotal { get; set; } + + [JsonPropertyName("fec_seconds")] + public long? FecSeconds { get; set; } + + [JsonPropertyName("tx_gem_frames_total")] + public long? TxGemFramesTotal { get; set; } + + [JsonPropertyName("tx_gem_bytes_total")] + public long? TxGemBytesTotal { get; set; } + + [JsonPropertyName("tx_gem_idle_frames_total")] + public long? TxGemIdleFramesTotal { get; set; } + + [JsonPropertyName("rx_gem_frames_total")] + public long? RxGemFramesTotal { get; set; } + + [JsonPropertyName("rx_gem_bytes_total")] + public long? RxGemBytesTotal { get; set; } + + [JsonPropertyName("rx_gem_frames_dropped")] + public long? RxGemFramesDropped { get; set; } + + [JsonPropertyName("omci_drop")] + public long? OmciDrop { get; set; } + + [JsonPropertyName("drop")] + public long? Drop { get; set; } + + [JsonPropertyName("rx_oversized_frames")] + public long? RxOversizedFrames { get; set; } + + [JsonPropertyName("allocations_total")] + public long? AllocationsTotal { get; set; } + + [JsonPropertyName("allocations_lost")] + public long? AllocationsLost { get; set; } + } + + public class GpeBridgePortSection + { + [JsonPropertyName("ibp_good")] + public long? IbpGood { get; set; } + + [JsonPropertyName("ibp_discard")] + public long? IbpDiscard { get; set; } + + [JsonPropertyName("ebp_good")] + public long? EbpGood { get; set; } + + [JsonPropertyName("ebp_discard")] + public long? EbpDiscard { get; set; } + + [JsonPropertyName("learning_discard")] + public long? LearningDiscard { get; set; } + } +} diff --git a/src/NetworkOptimizer.Web/wwwroot/css/app.css b/src/NetworkOptimizer.Web/wwwroot/css/app.css index ecca1f35a..5ca1cca61 100644 --- a/src/NetworkOptimizer.Web/wwwroot/css/app.css +++ b/src/NetworkOptimizer.Web/wwwroot/css/app.css @@ -17283,6 +17283,11 @@ a.isp-outage-ack:hover { } .isp-hop-name { + display: flex; + flex-wrap: wrap; + align-items: center; + column-gap: 0.5rem; + row-gap: 0.25rem; font-size: 0.85rem; font-weight: 500; color: var(--text-primary); @@ -17295,14 +17300,19 @@ a.isp-outage-ack:hover { } .isp-target-badge { - margin-left: 0.5rem; - padding: 0.05rem 0.4rem; + display: inline-flex; + align-items: center; + justify-content: center; + box-sizing: border-box; + padding: 0.15rem 0.4rem; + border: 0; border-radius: 4px; + font-family: inherit; font-size: 0.62rem; font-weight: 600; + line-height: 1; text-transform: uppercase; letter-spacing: 0.04em; - vertical-align: middle; } .isp-target-graded { diff --git a/src/NetworkOptimizer.Web/wwwroot/js/ont-charts.js b/src/NetworkOptimizer.Web/wwwroot/js/ont-charts.js index ff30aa870..b60d4d092 100644 --- a/src/NetworkOptimizer.Web/wwwroot/js/ont-charts.js +++ b/src/NetworkOptimizer.Web/wwwroot/js/ont-charts.js @@ -13,6 +13,9 @@ const RANGE_MS = { 0: 15*60000, 1: 3600000, 6: 6*3600000, 24: 86400000, 168: 7*8 let powerChart = null; let tempChart = null; +// FEC/BIP error-delta chart; its section stays hidden unless some ONT reports the counters. +let errorsChart = null; +let errorsSeriesByDevice = {}; let pollTimer = null; let currentRangeHours = 24; let windowOffset = 0; @@ -148,6 +151,10 @@ function updateVisibility() { if (vis) tempChart.showSeries(m.label); else tempChart.hideSeries(m.label); } + const es = errorsSeriesByDevice[m.id]; + if (errorsChart && es) { + es.forEach(n => vis ? errorsChart.showSeries(n) : errorsChart.hideSeries(n)); + } } catch (_) {} }); } @@ -193,6 +200,8 @@ async function loadAndUpdate() { powerChart.updateSeries(powerSeries, false); } if (tempChart) tempChart.updateSeries(tempSeries, false); + // Never let an errors-chart failure abort the rest of the refresh (badges, stats table). + try { await updateErrorsChart(data); } catch (_) {} updateVisibility(); lastData = data; @@ -205,6 +214,47 @@ async function loadAndUpdate() { const fmtDbm = v => v != null ? v.toFixed(2) : '-'; const fmtTemp = v => v != null ? v.toFixed(1) : '-'; +const fmtCount = v => v == null ? '' : v >= 1e6 ? (v / 1e6).toFixed(1) + 'M' : v >= 1e3 ? (v / 1e3).toFixed(1) + 'k' : String(Math.round(v)); + +// Create the errors chart on first use, so ONTs that never report FEC/BIP don't +// pay for an instance they'd never see. +async function ensureErrorsChartMounted() { + if (errorsChart) return; + const container = document.getElementById(containerId); + const errorsEl = container?.querySelector('.ont-errors-chart'); + if (!errorsEl) return; + errorsChart = new ApexCharts(errorsEl, { ...baseOpts(160, 'errors', fmtCount), series: [], colors: PALETTE }); + await errorsChart.render(); +} + +async function updateErrorsChart(data) { + const container = document.getElementById(containerId); + const section = container?.querySelector('.ont-errors-section'); + if (!section) return; + const withErrors = (data.devices || []).filter(d => + (d.data || []).some(p => p.fec != null || p.bip != null)); + if (!withErrors.length) { + section.style.display = 'none'; + errorsSeriesByDevice = {}; + return; + } + await ensureErrorsChartMounted(); + if (!errorsChart) return; + section.style.display = ''; + + errorsSeriesByDevice = {}; + const series = []; + withErrors.forEach(d => { + const pts = d.data || []; + const s = [ + { name: `${d.label} FEC`, data: pts.filter(p => p.fec != null).map(p => ({ x: new Date(p.time).getTime(), y: p.fec })) }, + { name: `${d.label} BIP`, data: pts.filter(p => p.bip != null).map(p => ({ x: new Date(p.time).getTime(), y: p.bip })) }, + ]; + errorsSeriesByDevice[d.id] = s.map(x => x.name); + series.push(...s); + }); + errorsChart.updateSeries(series, false); +} function renderStatsTable(container, showAll) { const el = container.querySelector('.ont-stats-table'); @@ -379,6 +429,7 @@ export async function mount(elId) { if (powerChart) { powerChart.destroy(); powerChart = null; } if (tempChart) { tempChart.destroy(); tempChart = null; } + if (errorsChart) { errorsChart.destroy(); errorsChart = null; } powerChart = new ApexCharts(powerEl, { ...baseOpts(220, 'dBm', v => v != null ? v.toFixed(1) + ' dBm' : '', { @@ -393,6 +444,9 @@ export async function mount(elId) { await powerChart.render(); await tempChart.render(); + // The FEC/BIP errors chart is mounted lazily (ensureErrorsChartMounted) only when + // an ONT actually reports those counters - ONTs without them never create it. + container.querySelectorAll('[data-range]').forEach(btn => { btn.addEventListener('click', () => selectPresetRange(container, parseInt(btn.dataset.range))); }); @@ -465,9 +519,11 @@ export function unmount() { if (fetchController) { fetchController.abort(); fetchController = null; } if (powerChart) { powerChart.destroy(); powerChart = null; } if (tempChart) { tempChart.destroy(); tempChart = null; } + if (errorsChart) { errorsChart.destroy(); errorsChart = null; } containerId = null; deviceMeta = []; visibility = {}; + errorsSeriesByDevice = {}; lastData = null; currentRangeHours = 24; windowOffset = 0; diff --git a/src/NetworkOptimizer.Web/wwwroot/js/sfp-charts.js b/src/NetworkOptimizer.Web/wwwroot/js/sfp-charts.js index 435609b7f..6cbdd1388 100644 --- a/src/NetworkOptimizer.Web/wwwroot/js/sfp-charts.js +++ b/src/NetworkOptimizer.Web/wwwroot/js/sfp-charts.js @@ -13,6 +13,14 @@ const RANGE_MS = { 0: 15*60000, 1: 3600000, 6: 6*3600000, 24: 86400000, 168: 7*8 let powerChart = null; let tempChart = null; +// Supplemental PON-layer charts (attached Network Optimizer Custom ONT configs); +// the whole section stays hidden unless some module has PON data. +let ponErrChart = null; +let ponGemChart = null; +let ponHostChart = null; +// Modules that currently have PON data, whether visible or not. The PON section and +// its detail table are shown only for the ones selected in the filter. +let ponCapableModules = []; let pollTimer = null; let currentRangeHours = 24; let windowOffset = 0; @@ -148,6 +156,53 @@ function updateVisibility() { else tempChart.hideSeries(m.label); } }); + // Fire-and-forget: rebuilds the PON charts/section for the selected modules. Kept + // off the synchronous path so a chart error can never break chip re-rendering. + refreshPonSection(); +} + +// Rebuild the PON section (charts + detail table) for the currently-selected PON +// modules, or hide it when none are selected. The section is shown BEFORE the charts +// are mounted/updated so ApexCharts sizes them correctly - a chart first rendered in a +// display:none container stays zero-size until its next update. +async function refreshPonSection() { + const container = document.getElementById(containerId); + const section = container?.querySelector('.sfp-pon-section'); + if (!section) return; + const visiblePon = ponCapableModules.filter(m => visibility[m.id] !== false); + if (!visiblePon.length) { section.style.display = 'none'; return; } + section.style.display = ''; + await ensurePonChartsMounted(); + if (!ponErrChart) return; + try { + const multi = visiblePon.length > 1; + const errSeries = [], gemSeries = [], hostSeries = []; + visiblePon.forEach(m => { + const pts = m.pon; + const prefix = multi ? `${m.label} ` : ''; + errSeries.push( + { name: `${prefix}BIP`, data: ponPoints(pts, 'bip') }, + { name: `${prefix}FEC`, data: ponPoints(pts, 'fec') }, + { name: `${prefix}FEC corrected`, data: ponPoints(pts, 'fecCorr') }, + { name: `${prefix}HEC`, data: ponPoints(pts, 'hec') }, + { name: `${prefix}GEM drops`, data: ponPoints(pts, 'gemDrop') }, + { name: `${prefix}Allocs lost`, data: ponPoints(pts, 'allocLost') }, + ); + gemSeries.push( + { name: `${prefix}RX frames`, data: ponPoints(pts, 'gemRx') }, + { name: `${prefix}TX frames`, data: ponPoints(pts, 'gemTx') }, + ); + hostSeries.push( + { name: `${prefix}FCS errors`, data: ponPoints(pts, 'lanFcs') }, + { name: `${prefix}TX drops`, data: ponPoints(pts, 'lanDrop') }, + { name: `${prefix}Buffer overflows`, data: ponPoints(pts, 'lanOvfl') }, + ); + }); + ponErrChart.updateSeries(errSeries, false); + ponGemChart.updateSeries(gemSeries, false); + ponHostChart.updateSeries(hostSeries, false); + renderPonDetails(container, visiblePon); + } catch (e) { /* leave the previous render if a chart update fails */ } } async function loadAndUpdate() { @@ -186,6 +241,8 @@ async function loadAndUpdate() { powerChart.updateSeries(powerSeries, false); } if (tempChart) tempChart.updateSeries(tSeries, false); + // Never let a PON chart failure abort the rest of the refresh (badges, stats table). + try { await updatePonCharts(data); } catch (_) {} updateVisibility(); lastData = data; @@ -198,6 +255,62 @@ async function loadAndUpdate() { const fmtDbm = v => v != null ? v.toFixed(2) : '-'; const fmtTemp = v => v != null ? v.toFixed(1) : '-'; +const fmtCount = v => v == null ? '' : v >= 1e6 ? (v / 1e6).toFixed(1) + 'M' : v >= 1e3 ? (v / 1e3).toFixed(1) + 'k' : String(Math.round(v)); + +// Same encoding PonLinkStateExtensions.ToInfluxValue uses for pon_link_status. +const PLOAM_LABELS = { + initial: 'Initializing (O1)', standby: 'Standby (O2)', serial_number: 'Authenticating (O3)', + ranging: 'Ranging (O4)', operation: 'Connected (O5)', popup: 'Signal Lost (O6)', + emergency_stop: 'Disabled (O7)', +}; + +function ponPoints(pts, key) { + return pts.filter(p => p[key] != null).map(p => ({ x: new Date(p.time).getTime(), y: p[key] })); +} + +// Create the three PON charts on first use. Kept out of mount() so setups without +// supplemental PON polling never pay for instances they'd never see. +async function ensurePonChartsMounted() { + if (ponErrChart) return; + const container = document.getElementById(containerId); + const ponErrEl = container?.querySelector('.sfp-pon-errors-chart'); + const ponGemEl = container?.querySelector('.sfp-pon-gem-chart'); + const ponHostEl = container?.querySelector('.sfp-pon-host-chart'); + if (!ponErrEl || !ponGemEl || !ponHostEl) return; + ponErrChart = new ApexCharts(ponErrEl, { ...baseOpts(160, 'errors', fmtCount), series: [], colors: PALETTE }); + ponGemChart = new ApexCharts(ponGemEl, { ...baseOpts(160, 'frames', fmtCount), series: [], colors: PALETTE }); + ponHostChart = new ApexCharts(ponHostEl, { ...baseOpts(160, 'errors', fmtCount), series: [], colors: PALETTE }); + await Promise.all([ponErrChart.render(), ponGemChart.render(), ponHostChart.render()]); +} + +async function updatePonCharts(data) { + ponCapableModules = (data.modules || []).filter(m => m.pon?.length); + await refreshPonSection(); +} + +function renderPonDetails(container, withPon) { + const el = container.querySelector('.sfp-pon-details'); + if (!el) return; + const fmtUp = s => s == null ? '-' + : `${Math.floor(s / 86400)}d ${Math.floor(s % 86400 / 3600)}h ${Math.floor(s % 3600 / 60)}m`; + const rows = withPon.map(m => { + const last = [...m.pon].reverse().find(p => p.state != null) || m.pon[m.pon.length - 1]; + const state = PLOAM_LABELS[last.state] || last.state || '-'; + const fec = last.dsFec == null && last.usFec == null ? '-' + : `${last.dsFec ? 'on' : 'off'} / ${last.usFec ? 'on' : 'off'}`; + return ` + ${escapeHtml(m.label)} + ${escapeHtml(state)} + ${last.onuId ?? '-'} + ${fec} + ${last.respTime ?? '-'} + ${fmtUp(last.uptime)} + `; + }).join(''); + el.innerHTML = `
+ + ${rows}
ModulePLOAM StateONU IDFEC DS / USResponse TimeONT Uptime
`; +} function renderStatsTable(container, showAll) { const el = container.querySelector('.sfp-stats-table'); @@ -363,6 +476,9 @@ export async function mount(elId) { if (powerChart) { powerChart.destroy(); powerChart = null; } if (tempChart) { tempChart.destroy(); tempChart = null; } + if (ponErrChart) { ponErrChart.destroy(); ponErrChart = null; } + if (ponGemChart) { ponGemChart.destroy(); ponGemChart = null; } + if (ponHostChart) { ponHostChart.destroy(); ponHostChart = null; } powerChart = new ApexCharts(powerEl, { ...baseOpts(200, 'dBm', v => v != null ? v.toFixed(1) + ' dBm' : '', { @@ -377,6 +493,10 @@ export async function mount(elId) { await powerChart.render(); await tempChart.render(); + // PON charts are mounted lazily (ensurePonChartsMounted) only once a module with + // supplemental PON polling actually reports data - the ~99% of SFP setups without + // it never create these instances. + container.querySelectorAll('[data-range]').forEach(btn => { btn.addEventListener('click', () => selectPresetRange(container, parseInt(btn.dataset.range))); }); @@ -469,9 +589,13 @@ export function unmount() { if (fetchController) { fetchController.abort(); fetchController = null; } if (powerChart) { powerChart.destroy(); powerChart = null; } if (tempChart) { tempChart.destroy(); tempChart = null; } + if (ponErrChart) { ponErrChart.destroy(); ponErrChart = null; } + if (ponGemChart) { ponGemChart.destroy(); ponGemChart = null; } + if (ponHostChart) { ponHostChart.destroy(); ponHostChart = null; } containerId = null; moduleMeta = []; visibility = {}; + ponCapableModules = []; lastData = null; currentRangeHours = 24; windowOffset = 0; diff --git a/tests/NetworkOptimizer.Web.Tests/NetOptCustomPonOntProviderTests.cs b/tests/NetworkOptimizer.Web.Tests/NetOptCustomPonOntProviderTests.cs new file mode 100644 index 000000000..efb57ab85 --- /dev/null +++ b/tests/NetworkOptimizer.Web.Tests/NetOptCustomPonOntProviderTests.cs @@ -0,0 +1,158 @@ +using FluentAssertions; +using NetworkOptimizer.Monitoring.Models; +using NetworkOptimizer.Web.Services.OntProviders; +using Xunit; + +namespace NetworkOptimizer.Web.Tests; + +public class NetOptCustomPonOntProviderTests +{ + // Real payload from the reference implementation (a GPON SFP stick relayed + // through the gateway), trimmed only in counter magnitudes. + private const string SamplePayload = """ + { + "lan": { "mode": 15, "link_status": 5, "phy_duplex": 1 }, + "lan_counters": { + "tx_frames": 671630919, "rx_frames": 850075595, + "tx_drop_events": 0, "rx_fcs_err": 3, "buffer_overflow": 0 + }, + "ploam": { "curr_state": 5, "previous_state": 4, "elapsed_msec": 4294791528 }, + "gtc_status": { + "ds_state": 3, "onu_id": 49, "ds_fec_enable": 0, "us_fec_enable": 0, + "onu_response_time": 34992 + }, + "gtc_counters": { + "bip": 7, "hec_error_corr": 0, "hec_error_uncorr": 1, + "bwmap_error_corr": 0, "bwmap_error_uncorr": 0, + "fec_error_corr": 0, "fec_words_corr": 12, "fec_words_uncorr": 2, + "fec_words_total": 1000, "fec_seconds": 0, + "tx_gem_frames_total": 848555332, "tx_gem_bytes_total": 1774937051, + "tx_gem_idle_frames_total": 1076427353, + "rx_gem_frames_total": 682156109, "rx_gem_bytes_total": 0, + "rx_gem_frames_dropped": 1, "omci_drop": 0, "drop": 1, + "rx_oversized_frames": 0, + "allocations_total": 1629046208, "allocations_lost": 20 + }, + "gpe_pon": { "ibp_good": 670798566, "ibp_discard": 4, "ebp_good": 848843235, "ebp_discard": 0, "learning_discard": 0 }, + "gpe_lan": { "ibp_good": 848843244, "ibp_discard": 0, "ebp_good": 670798569, "ebp_discard": 5, "learning_discard": 0 }, + "sfp_uptime_s": 358825 + } + """; + + [Fact] + public void ParsePayload_FullSample_ParsesAllSections() + { + var p = NetOptCustomPonOntProvider.ParsePayload(SamplePayload); + + p.Should().NotBeNull(); + p!.Error.Should().BeNull(); + p.Ploam!.CurrState.Should().Be(5); + p.Ploam.PreviousState.Should().Be(4); + p.GtcStatus!.OnuId.Should().Be(49); + p.GtcCounters!.Bip.Should().Be(7); + p.GtcCounters.AllocationsLost.Should().Be(20); + p.GpePon!.IbpDiscard.Should().Be(4); + p.GpeLan!.EbpDiscard.Should().Be(5); + p.LanCounters!.RxFcsErr.Should().Be(3); + p.SfpUptimeS.Should().Be(358825); + } + + [Fact] + public void MapToSupplemental_UsesStandardEncodingsAndCuratedFields() + { + var p = NetOptCustomPonOntProvider.ParsePayload(SamplePayload)!; + var s = NetOptCustomPonOntProvider.MapToSupplemental(p); + + // Standard concepts keep the ont measurement's encodings/semantics. + s.PonLinkStatus.Should().Be("operation"); + s.PonLinkStatusPrev.Should().Be("ranging"); + s.PloamStateRaw.Should().Be(5); + s.BipErrors.Should().Be(7); + s.FecErrors.Should().Be(2, "fec_errors is UNCORRECTABLE codewords, not corrected"); + s.FecCorrectedWords.Should().Be(12); + + s.HecUncorrected.Should().Be(1); + s.GemTxFrames.Should().Be(848555332); + s.GemTxIdleFrames.Should().Be(1076427353); + s.GemRxDropped.Should().Be(1); + s.AllocTotal.Should().Be(1629046208); + s.AllocLost.Should().Be(20); + s.GpePonIngressDiscard.Should().Be(4); + s.GpeLanEgressDiscard.Should().Be(5); + s.LanLinkStatus.Should().Be(5); + s.LanRxFcsErrors.Should().Be(3); + s.OnuId.Should().Be(49); + s.OnuResponseTime.Should().Be(34992); + s.DsFecEnabled.Should().Be(0); + s.SfpUptimeS.Should().Be(358825); + } + + [Fact] + public void MapToOntStats_PopulatesStandardOntFields() + { + var p = NetOptCustomPonOntProvider.ParsePayload(SamplePayload)!; + var stats = NetOptCustomPonOntProvider.MapToOntStats(p); + + stats.PonLinkStatus.Should().Be(PonLinkState.Operation); + stats.FecErrors.Should().Be(2); + stats.BipErrors.Should().Be(7); + } + + [Fact] + public void ParsePayload_ErrorShape_SurfacesErrorCode() + { + var p = NetOptCustomPonOntProvider.ParsePayload( + """{ "error": "sfp_unreachable", "message": "Could not reach SFP" }"""); + + p.Should().NotBeNull(); + p!.Error.Should().Be("sfp_unreachable"); + p.Message.Should().Be("Could not reach SFP"); + } + + [Fact] + public void ParsePayload_EmptyObject_AllSectionsNull() + { + var p = NetOptCustomPonOntProvider.ParsePayload("{}"); + + p.Should().NotBeNull(); + p!.Ploam.Should().BeNull(); + p.GtcCounters.Should().BeNull(); + p.SfpUptimeS.Should().BeNull(); + + var s = NetOptCustomPonOntProvider.MapToSupplemental(p); + s.PonLinkStatus.Should().BeNull(); + s.BipErrors.Should().BeNull(); + s.FecErrors.Should().BeNull(); + } + + [Fact] + public void ParsePayload_StringNumbers_AreTolerated() + { + var p = NetOptCustomPonOntProvider.ParsePayload( + """{ "ploam": { "curr_state": "5" }, "gtc_status": { "onu_id": "49" } }"""); + + p!.Ploam!.CurrState.Should().Be(5); + p.GtcStatus!.OnuId.Should().Be(49); + } + + [Fact] + public void ParsePayload_Garbage_ReturnsNull() + { + NetOptCustomPonOntProvider.ParsePayload("not json at all").Should().BeNull(); + NetOptCustomPonOntProvider.ParsePayload("").Should().BeNull(); + } + + [Theory] + [InlineData(1, PonLinkState.Initial)] + [InlineData(4, PonLinkState.Ranging)] + [InlineData(5, PonLinkState.Operation)] + [InlineData(6, PonLinkState.Popup)] + [InlineData(7, PonLinkState.EmergencyStop)] + [InlineData(0, PonLinkState.Unknown)] + [InlineData(8, PonLinkState.Unknown)] + [InlineData(null, PonLinkState.Unknown)] + public void ToPonLinkState_MapsItuStateNumbers(int? raw, PonLinkState expected) + { + NetOptCustomPonOntProvider.ToPonLinkState(raw).Should().Be(expected); + } +} diff --git a/tests/NetworkOptimizer.Web.Tests/OntAlertEvaluatorTests.cs b/tests/NetworkOptimizer.Web.Tests/OntAlertEvaluatorTests.cs index b5f67b31d..424a76e21 100644 --- a/tests/NetworkOptimizer.Web.Tests/OntAlertEvaluatorTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/OntAlertEvaluatorTests.cs @@ -98,6 +98,82 @@ await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bus.Events.Should().NotContain(e => e.EventType == TempEvent); } + [Fact] + public async Task BipSpike_FecDisabled_UsesStrictThreshold() + { + var (evaluator, bus) = Create(); + + // FEC off: BIP is uncorrected data loss, so the strict 25-error threshold applies. + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 0, fecEnabled: false); + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 50, fecEnabled: false); + + var bip = bus.Events.Where(e => e.EventType == "ont.bip_errors").ToList(); + bip.Should().HaveCount(1); + bip[0].Source.Should().Be("ont"); + bip[0].MetricValue.Should().Be(50); + bip[0].ThresholdValue.Should().Be(25); + } + + [Fact] + public async Task BipSpike_FecEnabled_UsesRelaxedThreshold() + { + var (evaluator, bus) = Create(); + + // FEC on: BIP counts pre-FEC line errors FEC corrects, so a 300-error step (below the + // relaxed 1000 threshold) must NOT alert, while a larger 1100-error step does. + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 0, fecEnabled: true); + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 300, fecEnabled: true); + bus.Events.Should().NotContain(e => e.EventType == "ont.bip_errors"); + + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 1400, fecEnabled: true); + var bip = bus.Events.Where(e => e.EventType == "ont.bip_errors").ToList(); + bip.Should().HaveCount(1); + bip[0].MetricValue.Should().Be(1100); + bip[0].ThresholdValue.Should().Be(1000); + } + + [Fact] + public async Task BipCounterReset_DoesNotFakeSpike() + { + var (evaluator, bus) = Create(); + + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 5000, fecEnabled: false); + // ONT reboots; counter resets below the prior value -> negative step -> no alert. + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, null, bipErrors: 10, fecEnabled: false); + + bus.Events.Should().NotContain(e => e.EventType == "ont.bip_errors"); + } + + [Fact] + public async Task FecDisabled_EvaluatesHecNotFec() + { + var (evaluator, bus) = Create(); + + // With FEC disabled, a large FEC delta must be ignored and HEC drives the codeword alert. + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, fecErrors: 0, + hecErrors: 0, fecEnabled: false); + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, fecErrors: 100000, + hecErrors: 500, fecEnabled: false); + + bus.Events.Should().NotContain(e => e.EventType == "ont.fec_errors"); + var hec = bus.Events.Where(e => e.EventType == "ont.hec_errors").ToList(); + hec.Should().HaveCount(1); + hec[0].MetricValue.Should().Be(500); + } + + [Fact] + public async Task FecEnabledOrUnknown_EvaluatesFecNotHec() + { + var (evaluator, bus) = Create(); + + // Default (fecEnabled unknown/null, the standalone-ONT case): FEC drives the alert, HEC is ignored. + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, fecErrors: 0, hecErrors: 0); + await evaluator.EvaluateAsync(1, "ONT", SafeRx, PonLinkState.Operation, fecErrors: 2000, hecErrors: 9999); + + bus.Events.Should().NotContain(e => e.EventType == "ont.hec_errors"); + bus.Events.Count(e => e.EventType == "ont.fec_errors").Should().Be(1); + } + private sealed class CapturingBus : IAlertEventBus { public List Events { get; } = new(); From 8942e457e8496830396105b23c887d1cf489a786 Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:50:55 -0500 Subject: [PATCH 06/23] ISP Health: surface your direct peering in Transit Health (#1031) * ISP Health: grade direct-peered internet destinations as an IX Peering entry in Transit Health When internet targets are reached directly over the access ISP's peering/IX (no transit ASN on the forward path, and a small best-case delta beyond the first clean ISP hop), grade their measured end-to-end quality as a synthetic "IX Peering" transit entry instead of leaving the Transit dimension averaging against the neutral-100 fill. Selection requires BOTH arms: no transit on the path (AS-path, reusing the routes-through ancestry) AND best-case delta under 10 ms. Far-but-untransited peers and low-latency-but-transited destinations are both excluded. No entry appears where every destination crosses a transit provider. Adds two scorer tests. * ISP Health: grade IX Peering per-destination and average, not pooled Pooling all peered destinations' samples into one series manufactured cross-target variance (targets at different RTT baselines look like one swinging series), craters the stability sub-score, and let one target's jitter tail dominate the pooled P95. Grade each peered destination on its own series and average the grades, so one flapping peer counts 1/N. Log each member's grade at Debug for diagnosis. * ISP Health: IX Peering proximity jitter absolution + weight-desc network ordering Proximity absolution: when peered destinations reach the internet at a fraction of the median transit ASN's RTT, buy back part of the IX Peering jitter penalty, scaled by the RTT advantage and bounded to jitter's own weighted share of the grade (never masks loss, stability, or a real congestion penalty). Where transit is as close as peering, advantage < 1 and no bonus applies. Ordering: sort the transit/IX entries by involvement weight descending (nearer network breaks ties) for both the Transit Health factor list and Networks on Your Path; Access renders first as before. * ISP Health: Transit Health = plain involvement-weighted average (drop neutral-100 blend) The dimension blended each transit ASN's uninvolved fraction toward a neutral 100 (effective = w*own + (1-w)*100), which inflated Transit Health above every real entry (e.g. entries 88/73/89 -> 90) and re-introduced the phantom-100 the IX Peering entry was meant to eliminate - masking a real congested-but-off-path transit up toward 100. Replace with a plain involvement-weighted average of the entries' own scores: the dimension now always lands within its entries' range, and a congested off-path transit dings it lightly (0.25 floor) rather than hiding at 100. Floor unchanged (return-path/failover accountability). --- .../Shared/Monitoring/IspHealthPanel.razor | 2 +- .../Monitoring/IspHealth/IspHealthModels.cs | 6 +- .../Monitoring/IspHealth/IspHealthScorer.cs | 209 ++++++++++++++++-- .../IspHealth/IspHealthScorerTests.cs | 75 ++++++- 4 files changed, 269 insertions(+), 23 deletions(-) diff --git a/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor b/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor index 7a362a819..97d04355b 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor @@ -501,7 +501,7 @@ else @InvolvementPie(asn.InvolvementWeight ?? 1.0, asn.InvolvementTooltip!) } -
@(isIspAsn ? "ISP network" : "Transit") @(asn.AsnNumber > 0 ? $"· AS{asn.AsnNumber}" : "")
+
@(isIspAsn ? "ISP network" : asn.AsnNumber < 0 ? "Direct peering" : "Transit") @(asn.AsnNumber > 0 ? $"· AS{asn.AsnNumber}" : "")
@(asn.OverallScore?.ToString() ?? "--") diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs index 9b500049f..8711a36dd 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs @@ -124,9 +124,9 @@ public class IspAsnHealth public int InvolvementHostTotal { get; set; } /// - /// The 0.25-1.0 weight this ASN's own score carries in Transit Health; the remaining (1 - weight) - /// blends to a neutral 100, so a transit carrying none of your hosts can't drag the dimension. - /// Null when involvement isn't shown. + /// The 0.25-1.0 weight this ASN's own score carries in the involvement-weighted Transit Health + /// average. Floored at 0.25 so an off-path transit still counts - it may be on your return path or a + /// failover route - but only lightly. Null when involvement isn't shown. /// public double? InvolvementWeight { get; set; } diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs index a869c5588..5f2128f0e 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs @@ -94,6 +94,27 @@ public IspHealthReport Score(IspHealthInputs inputs, AccessProfile profile) var transitAsns = GradeTransitAsns(inputs.TransitAsnSeries, inputs.DestinationSeries, inputs.HopOrderKnown, inputs.CongestionEvents, jitterFloor, accessMedianRtt, inputs.InternetMedianDeltaMs); + // IX Peering: destinations reached over the access ISP's own peering/IX (see + // IxPeeringMaxBestCaseDeltaMs) are graded end-to-end and inserted as one synthetic transit + // entry, so the REAL measured peering quality carries Transit Health instead of the neutral-100 + // fill an otherwise-empty (purely peered) transit dimension would average against. Nothing + // qualifies where every destination crosses a transit provider (e.g. a rural backhaul), and no + // entry is added - the prior behavior stands. + var peeringReached = SelectPeeringReachedDestinations(inputs); + if (peeringReached.Count > 0) + { + // Median RTT of the real transit ASNs (before the IX entry joins the list). Peering that + // delivers the internet far closer than this earns back some of its jitter - see GradeIxPeering. + var transitReferenceRtt = SeriesStats.Median( + transitAsns.Where(a => a.MeanRttMs.HasValue).Select(a => a.MeanRttMs!.Value).ToList()); + var ixGrade = GradeIxPeering(peeringReached, inputs.CongestionEvents, jitterFloor, accessMedianRtt, + inputs.InternetMedianDeltaMs, transitReferenceRtt); + transitAsns.Insert(0, ixGrade); + _logger?.LogDebug( + "ISP Health: IX Peering entry from {N} direct-peered destination(s) [{Targets}] -> grade {Grade}", + peeringReached.Count, string.Join(", ", peeringReached.Select(d => d.AsnName)), ixGrade.OverallScore); + } + // Arm 4: each transit ASN's "internet host involvement" - how many monitored internet // destinations are proven to route through it (routes-through gated on stored ancestry). // Feeds the involvement-weighted Transit dimension below. Zero for every ASN when transit @@ -105,7 +126,14 @@ public IspHealthReport Score(IspHealthInputs inputs, AccessProfile profile) var transitReachByAsn = transitHopIpsByAsn.ToDictionary( kv => kv.Key, kv => inputs.HopOrderKnown ? inputs.DestinationSeries.Count(d => RoutesThrough(d.AncestorIps, kv.Value)) : 0); - var transitMaxReach = transitReachByAsn.Count > 0 ? transitReachByAsn.Values.Max() : 0; + // Reach drives the involvement weight. Real transit ASNs use the routes-through count above; + // the synthetic IX Peering entry carries exactly the direct-peered destinations it was built + // from, so in a peering-dominant path it holds max reach (weight ~1) and its real grade - not + // the neutral 100 - carries the dimension. + int ReachOf(IspAsnHealth a) => a.AsnNumber == IxPeeringAsn + ? peeringReached.Count + : transitReachByAsn.TryGetValue(a.AsnNumber, out var rc) ? rc : 0; + var transitMaxReach = transitAsns.Count > 0 ? transitAsns.Max(ReachOf) : 0; // Attribution is "known" when we have the ancestry to prove routes-through AND destinations to // test against. Then reach == 0 is a TRUE zero (a peered site whose destinations cross no @@ -117,7 +145,7 @@ public IspHealthReport Score(IspHealthInputs inputs, AccessProfile profile) { a.ShowInvolvement = transitAttributionKnown; a.InvolvementHostTotal = inputs.DestinationSeries.Count; - a.InvolvementReach = transitReachByAsn.TryGetValue(a.AsnNumber, out var rc) ? rc : 0; + a.InvolvementReach = ReachOf(a); a.InvolvementWeight = transitAttributionKnown ? (transitMaxReach > 0 ? TransitInvolvementFloor + (1 - TransitInvolvementFloor) * ((double)a.InvolvementReach / transitMaxReach) @@ -125,6 +153,15 @@ public IspHealthReport Score(IspHealthInputs inputs, AccessProfile profile) : null; } + // Display order for both the Transit Health factor list and the Networks on Your Path card: + // the IX Peering entry and the transit ASNs sorted by involvement weight descending (the + // networks carrying most of your traffic first), the nearer network breaking ties. The Access + // dimension renders separately and always leads the Networks card. + transitAsns = transitAsns + .OrderByDescending(a => a.InvolvementWeight ?? 0.0) + .ThenBy(a => a.MeanRttMs ?? double.MaxValue) + .ToList(); + // Every ISP hop is graded; the dimension averages them all. Each hop's jitter is // absolved per-hop and routes-through-gated (a transit ASN or deeper ISP hop only // absolves a hop it is proven downstream of), so a divergent clean transit can't @@ -1096,6 +1133,135 @@ private List GradeTransitAsns( return grades; } + /// + /// Selects the monitored internet destinations reached over the access ISP's own peering/IX rather + /// than a transit provider: the forward path crosses no transit ASN AND the best-case delta beyond + /// the first clean ISP hop is under . Requires + /// per-destination ancestry (hop order + a non-empty ancestor set); a destination we can't place on + /// the path is never assumed peered. Empty when nothing qualifies. + /// + private List SelectPeeringReachedDestinations(IspHealthInputs inputs) + { + if (!inputs.HopOrderKnown) return new List(); + + // Best-case (min) RTT of the first clean ISP hop: the access baseline the delta subtracts, so + // the threshold measures the incremental peering-path latency, not the access medium's own. + var firstHopBestCase = inputs.FirstHopSeries + .Where(s => s.RttAvgMs.HasValue).Select(s => s.RttAvgMs!.Value) + .DefaultIfEmpty(double.NaN).Min(); + var baseline = double.IsNaN(firstHopBestCase) ? 0.0 : firstHopBestCase; + + double? BestCaseDeltaMs(AsnSeries d) + { + var min = d.Samples.Where(s => s.RttAvgMs.HasValue).Select(s => s.RttAvgMs!.Value) + .DefaultIfEmpty(double.NaN).Min(); + return double.IsNaN(min) ? (double?)null : Math.Max(0, min - baseline); + } + + // Min (best case), not mean: a peered-but-flapping anycast (a low floor with occasional spikes) + // is still peering - the flapping belongs in the grade, not the peering/transit classification. + return inputs.DestinationSeries + .Where(d => d.AncestorIps.Count > 0 + && !inputs.TransitAsnSeries.Any(t => RoutesThrough(d.AncestorIps, t.HopIps)) + && BestCaseDeltaMs(d) is double delta && delta < IxPeeringMaxBestCaseDeltaMs) + .ToList(); + } + + /// + /// Grades the direct-peered destinations (from ) as one + /// synthetic "IX Peering" transit entry. Each destination is graded on ITS OWN series on the same + /// jitter/loss/stability/reach basis as a real transit ASN, and the grades are AVERAGED - the series are + /// never pooled. Pooling samples across targets at different RTT baselines (a flat ~2 ms peer beside a + /// flat ~6 ms peer) manufactures cross-target variance that craters the stability sub-score, and lets a + /// single target's jitter tail dominate the pooled P95; per-peer grading keeps each target coherent and + /// lets one degraded destination count only 1/N. Distance stays low (peered), so each reach ceiling + /// barely bites and the grade is driven by jitter and loss. A proximity absolution then buys back a + /// fraction of the jitter penalty when peering delivers the internet far closer than the transit + /// alternative (). Per-member grades are logged at Debug. + /// + private IspAsnHealth GradeIxPeering( + List peeringReached, + List congestionEvents, + double? jitterFloorMs, + double? accessBaselineRtt, + double? internetMedianDeltaMs, + double? transitReferenceRttMs) + { + var perPeer = peeringReached + .Select(d => GradeAsn(d, congestionEvents, jitterFloorMs, accessBaselineRtt, internetMedianDeltaMs)) + .ToList(); + + foreach (var p in perPeer) + { + _logger?.LogDebug( + "ISP Health: IX Peering member {Name} -> {Overall} (stability {Stab}, jitter {Jit} @ P95 {P95j} ms, loss {Loss} @ {LossPct}%, reach ceiling {Reach})", + p.AsnName, p.OverallScore, p.LatencyStabilityScore, p.JitterScore, FormatMsOrNull(p.P95JitterMs), + p.LossScore, p.LossPct?.ToString("0.###", CultureInfo.InvariantCulture) ?? "n/a", p.ReachLatencyScore); + } + + double? AvgOf(Func sel) + { + var vals = perPeer.Select(sel).Where(v => v.HasValue).Select(v => v!.Value).ToList(); + return vals.Count > 0 ? vals.Average() : null; + } + int? AvgIntOf(Func sel) + { + var vals = perPeer.Select(sel).Where(v => v.HasValue).Select(v => v!.Value).ToList(); + return vals.Count > 0 ? (int?)Math.Round(vals.Average()) : null; + } + + var meanRtt = AvgOf(p => p.MeanRttMs); + var jitterScore = AvgIntOf(p => p.JitterScore); + var overall = AvgIntOf(p => p.OverallScore); + + // Proximity jitter absolution. A few ms of jitter on a peered path that reaches the internet in a + // fraction of the transit RTT is cheap - the proximity is itself a quality win, and much of that + // jitter is anycast-endpoint ICMP handling, not path. Scale a give-back by how much closer peering + // is than the median transit ASN, and bound it to jitter's OWN weighted share of the grade, so it + // can only ever cancel jitter's cost - never loss, stability, or a real congestion penalty. + if (transitReferenceRttMs is double tr && meanRtt is double ix && ix > 0 + && jitterScore is int js && js < 100 && overall is int baseOverall) + { + var advantage = tr / ix; // 2.0 => peering reaches the internet at half the transit RTT + var alpha = ScoreCurve.Interpolate(advantage, (1.0, 0), (1.5, 0.25), (2.0, 0.45), (3.0, 0.6)); + if (alpha > 0) + { + var qualityWeight = _options.AsnLatencyStabilityWeight + _options.AsnJitterWeight + + _options.AsnLossWeight + _options.AsnCongestionWeight; + var jitterShare = qualityWeight > 0 ? _options.AsnJitterWeight / qualityWeight : 0; + var recovered = alpha * jitterShare * (100 - js); + overall = Math.Min(100, (int)Math.Round(baseOverall + recovered)); + _logger?.LogDebug( + "ISP Health: IX Peering proximity absolution - peering {Ix:0.0} ms vs transit ~{Tr:0.0} ms ({Adv:0.0}x), alpha {A:0.00} -> +{Rec:0.0} pts ({Base} -> {Ov})", + ix, tr, advantage, alpha, recovered, baseOverall, overall); + } + } + + return new IspAsnHealth + { + AsnNumber = IxPeeringAsn, + AsnName = "IX Peering", + TargetIds = peeringReached.SelectMany(d => d.TargetIds).Distinct().ToList(), + MedianRttMs = AvgOf(p => p.MedianRttMs), + MeanRttMs = meanRtt, + P95RttMs = AvgOf(p => p.P95RttMs), + MedianJitterMs = AvgOf(p => p.MedianJitterMs), + P95JitterMs = AvgOf(p => p.P95JitterMs), + LossPct = AvgOf(p => p.LossPct), + ReachDeltaMs = AvgOf(p => p.ReachDeltaMs), + RawJitterMs = AvgOf(p => p.RawJitterMs), + LatencyStabilityScore = AvgIntOf(p => p.LatencyStabilityScore), + JitterScore = jitterScore, + LossScore = AvgIntOf(p => p.LossScore), + ReachLatencyScore = AvgIntOf(p => p.ReachLatencyScore), + CongestionScore = AvgIntOf(p => p.CongestionScore), + // Average of the per-member grades (one flapping peer moves it 1/N), then lifted by the + // proximity absolution above. + OverallScore = overall, + CongestionEventCount = perPeer.Sum(p => p.CongestionEventCount) + }; + } + private List GradeIspHops( List ispHopSeries, List transitSeries, @@ -1321,17 +1487,31 @@ private static IspScoreDimension BuildDimension(string name, double weight, List /// private const double TransitInvolvementFloor = 0.25; - /// Neutral baseline the uninvolved fraction of a transit ASN's score blends toward: a - /// transit carrying none of your hosts can't hurt Transit Health (it contributes ~100). - private const double TransitNeutralBaseline = 100.0; + /// + /// A monitored internet destination is treated as reached over the access ISP's own peering/IX + /// (rather than a transit provider) when BOTH hold: its forward path crosses no transit ASN, and + /// its best-case (min) RTT delta beyond the first clean ISP hop is under this. The AS-path arm + /// alone would count a low-latency destination that still traverses transit (e.g. a nearby transit + /// PoP); the latency arm alone would count a peer sitting behind a long hidden L2 haul (an IX-local + /// AS-path that's still +15 ms). Both together isolate genuine direct peering. It's a delta beyond + /// the access hop, so the access technology's own latency is already subtracted and the same + /// threshold holds for fiber, cable, cellular, or satellite. + /// + private const double IxPeeringMaxBestCaseDeltaMs = 10.0; + + /// Synthetic ASN number for the entry. Negative so every + /// display path gated on AsnNumber > 0 (the "· ASn" suffix, BGP toolkit links) skips it. + private const int IxPeeringAsn = -1; /// - /// Builds a transit-style ASN dimension. Each ASN carries its - /// (0.25-1.0, set upstream from internet-host involvement); its effective contribution is - /// weight * own + (1 - weight) * , and the dimension score is - /// the involvement-weighted average of those - so the transit you actually use dominates and a - /// side-path transit's problems don't drag the number. When no ASN has involvement set (no - /// attribution), it's the plain average and no fraction icon is shown. + /// Builds a transit-style ASN dimension: a plain involvement-weighted average of the entries' own + /// scores. Each carries its (0.25-1.0, set upstream from + /// internet-host involvement), so the networks you actually use dominate and a side-path transit + /// counts only lightly - but its REAL score, never a neutral fill. The dimension therefore always + /// lands within the range of its entries (no synthetic baseline can lift it above every one), and a + /// congested off-path transit still dings it a little rather than being masked toward 100 - it may be + /// on your return path or a failover route. When no ASN has involvement set (no attribution), it's + /// the plain average and no fraction icon is shown. /// private static IspScoreDimension BuildAsnDimension(string name, double weight, List asns) { @@ -1357,12 +1537,7 @@ private static IspScoreDimension BuildAsnDimension(string name, double weight, L { var wsum = scored.Sum(a => a.InvolvementWeight ?? 1.0); score = wsum > 0 - ? (int)Math.Round(scored.Sum(a => - { - var w = a.InvolvementWeight ?? 1.0; - var effective = w * a.OverallScore!.Value + (1 - w) * TransitNeutralBaseline; - return w * effective; - }) / wsum) + ? (int)Math.Round(scored.Sum(a => (a.InvolvementWeight ?? 1.0) * a.OverallScore!.Value) / wsum) : (int)Math.Round(scored.Average(a => a.OverallScore!.Value)); } return new IspScoreDimension { Name = name, Score = score, Weight = weight, Factors = factors }; diff --git a/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs b/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs index d2b05ec9c..fb72145d7 100644 --- a/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs @@ -1160,8 +1160,79 @@ public void Transit_asns_with_no_attributable_hosts_are_floored_and_labeled() BuildInputs(transit: Transits(), destinations: new List { peeredDest }, hopOrderKnown: true), Gpon) .TransitDimension; - dim.Factors.Should().OnlyContain(f => f.InvolvementTooltip != null && f.InvolvementTooltip.Contains("25%"), - "complete ancestry showing zero transit involvement floors and labels every transit ASN"); + // The two transit ASNs carry zero hosts, so both are floored at 25% and labeled. + dim.Factors.Where(f => f.Name != "IX Peering").Should() + .OnlyContain(f => f.InvolvementTooltip != null && f.InvolvementTooltip.Contains("25%"), + "complete ancestry showing zero transit involvement floors and labels every transit ASN"); + // And because the destination is reached directly (no transit, low delta), its measured quality + // is surfaced as the IX Peering entry carrying full weight - not the neutral-100 fill. + dim.Factors.Should().ContainSingle(f => f.Name == "IX Peering") + .Which.InvolvementTooltip.Should().Contain("100% weight"); + } + + [Fact] + public void Ix_peering_entry_requires_both_low_delta_and_no_transit_on_path() + { + // Only the direct-peered destination becomes the synthetic IX Peering entry. The other two are + // each excluded by one arm of the AND rule: + // - ViaTransit: low RTT, but its path crosses a transit ASN (the "fast but not peering" case). + // - FarPeer: crosses no transit, but a large best-case delta (peering behind a hidden L2 haul). + var transit = new List + { + new() { AsnNumber = 64500, AsnName = "Transit", TargetIds = { "t" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 10, 0.5), HopIps = { "20.0.0.1" } } + }; + var peered = new AsnSeries + { + AsnNumber = 13335, AsnName = "Peered", TargetIds = { "peered" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 4, 0.3), HopIps = { "30.0.0.1" }, + AncestorIps = { "10.0.0.1" } // access ISP hop only - crosses no transit + }; + var viaTransit = new AsnSeries + { + AsnNumber = 15169, AsnName = "ViaTransit", TargetIds = { "via" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 4, 0.3), HopIps = { "31.0.0.1" }, + AncestorIps = { "10.0.0.1", "20.0.0.1" } // low RTT but routes through the transit ASN + }; + var farPeer = new AsnSeries + { + AsnNumber = 54113, AsnName = "FarPeer", TargetIds = { "far" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 20, 0.3), HopIps = { "32.0.0.1" }, + AncestorIps = { "10.0.0.1" } // crosses no transit, but ~18 ms beyond the access hop + }; + + var dim = new IspHealthScorer(Options).Score( + BuildInputs(transit: transit, destinations: new List { peered, viaTransit, farPeer }, + hopOrderKnown: true), Gpon).TransitDimension; + + var ix = dim.Factors.Should().ContainSingle(f => f.Name == "IX Peering").Which; + ix.Score.Should().NotBeNull("the peered destination's own measured quality drives the entry"); + ix.InvolvementTooltip.Should().Contain("1 of 3 internet targets", + "only the direct-peered destination qualifies; the transit-crossing and far ones are excluded"); + } + + [Fact] + public void Ix_peering_entry_is_absent_when_no_destination_is_directly_peered() + { + // Every destination crosses the transit ASN (a rural-style backhaul), so no IX Peering entry is + // synthesized and Transit Health grades on the real transit alone - the prior behavior stands. + var transit = new List + { + new() { AsnNumber = 64500, AsnName = "Transit", TargetIds = { "t" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 10, 0.5), HopIps = { "20.0.0.1" } } + }; + var viaTransit = new AsnSeries + { + AsnNumber = 15169, AsnName = "ViaTransit", TargetIds = { "via" }, + Samples = TestSeries.Flat(TestSeries.Start, Day, 12, 0.3), HopIps = { "31.0.0.1" }, + AncestorIps = { "10.0.0.1", "20.0.0.1" } + }; + + var dim = new IspHealthScorer(Options).Score( + BuildInputs(transit: transit, destinations: new List { viaTransit }, + hopOrderKnown: true), Gpon).TransitDimension; + + dim.Factors.Should().NotContain(f => f.Name == "IX Peering"); } [Fact] From 507b225a5aed670e1aebc77328824f1aa7e590e8 Mon Sep 17 00:00:00 2001 From: Jim Strang Date: Mon, 20 Jul 2026 10:59:35 -0400 Subject: [PATCH 07/23] Technitium DNS: prefer /api/status for provider detection (#1023) --- .../Dns/ThirdPartyDnsDetector.cs | 63 ++++++++- .../Dns/ThirdPartyDnsDetectorTests.cs | 122 +++++++++++++++++- 2 files changed, 182 insertions(+), 3 deletions(-) diff --git a/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs b/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs index 86715267d..4a4ba64df 100644 --- a/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs +++ b/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs @@ -994,11 +994,15 @@ private async Task ProbeTechnitiumDnsAsync(string ipAddress, int? customPo /// private async Task TryProbeTechnitiumDnsEndpointAsync(string baseUrl, int timeoutSeconds = 3) { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); + + if (await TryProbeTechnitiumStatusAsync(baseUrl, cts.Token)) + return true; + try { _logger.LogDebug("Probing Technitium DNS at {Url}", baseUrl); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); var response = await _httpClient.GetAsync(baseUrl, cts.Token); if (!response.IsSuccessStatusCode) @@ -1024,6 +1028,63 @@ private async Task TryProbeTechnitiumDnsEndpointAsync(string baseUrl, int } } + /// + /// Probe the Technitium status API within the parent endpoint's timeout budget. + /// + private async Task TryProbeTechnitiumStatusAsync(string baseUrl, CancellationToken cancellationToken) + { + var statusUrl = $"{baseUrl}/api/status"; + + try + { + _logger.LogDebug("Probing Technitium DNS status API at {Url}", statusUrl); + + var response = await _httpClient.GetAsync(statusUrl, cancellationToken); + + if (!response.IsSuccessStatusCode) + return false; + + var content = await response.Content.ReadAsStringAsync(cancellationToken); + using var document = JsonDocument.Parse(content); + var root = document.RootElement; + + if (root.ValueKind != JsonValueKind.Object || + !root.TryGetProperty("status", out var status) || + status.ValueKind != JsonValueKind.String || + !string.Equals(status.GetString(), "ok", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var hasDefaultCredentials = root.TryGetProperty("hasDefaultCredentials", out var defaultCredentials) && + defaultCredentials.ValueKind is JsonValueKind.True or JsonValueKind.False; + var hasSsoEnabled = root.TryGetProperty("ssoEnabled", out var ssoEnabled) && + ssoEnabled.ValueKind is JsonValueKind.True or JsonValueKind.False; + + return hasDefaultCredentials && hasSsoEnabled; + } + catch (TaskCanceledException) + { + _logger.LogDebug("Technitium DNS status API probe to {Url} timed out", statusUrl); + return false; + } + catch (HttpRequestException ex) + { + _logger.LogDebug("Technitium DNS status API probe to {Url} failed: {Message}", statusUrl, ex.Message); + return false; + } + catch (JsonException ex) + { + _logger.LogDebug("Technitium DNS status API probe to {Url} returned invalid JSON: {Message}", statusUrl, ex.Message); + return false; + } + catch (Exception ex) + { + _logger.LogDebug("Technitium DNS status API probe to {Url} error: {Type} - {Message}", statusUrl, ex.GetType().Name, ex.Message); + return false; + } + } + private async Task TryProbeTechnitiumDnsEndpointAsync(string ipAddress, int port, bool useHttps = false) { var scheme = useHttps ? "https" : "http"; diff --git a/tests/NetworkOptimizer.Audit.Tests/Dns/ThirdPartyDnsDetectorTests.cs b/tests/NetworkOptimizer.Audit.Tests/Dns/ThirdPartyDnsDetectorTests.cs index f61c3bfad..0fc5c2998 100644 --- a/tests/NetworkOptimizer.Audit.Tests/Dns/ThirdPartyDnsDetectorTests.cs +++ b/tests/NetworkOptimizer.Audit.Tests/Dns/ThirdPartyDnsDetectorTests.cs @@ -171,6 +171,93 @@ public async Task DetectThirdPartyDnsAsync_TechnitiumDnsDetected_FlagsProviderAs result[0].DnsProviderName.Should().Be("Technitium DNS"); } + [Fact] + public async Task DetectThirdPartyDnsAsync_TechnitiumStatusApiDetected_FlagsProviderAsTechnitiumDns() + { + const string baseUrl = "http://10.10.20.30:5380"; + var httpClient = CreateRequestAwareMockHttpClient(request => + { + if (request.RequestUri?.AbsoluteUri == $"{baseUrl}/api/status") + { + return new HttpResponseMessage + { + StatusCode = HttpStatusCode.OK, + Content = new StringContent("""{"hasDefaultCredentials":false,"ssoEnabled":true,"server":"technitium.example.local","status":"ok"}""") + }; + } + + return new HttpResponseMessage { StatusCode = HttpStatusCode.NotFound }; + }); + var detector = CreateDetector(httpClient); + + var result = await detector.DetectThirdPartyDnsAsync(CreateTechnitiumTestNetworks(), customUrl: baseUrl); + + result.Should().ContainSingle(); + result[0].IsTechnitiumDns.Should().BeTrue(); + result[0].DnsProviderName.Should().Be("Technitium DNS"); + } + + [Theory] + [InlineData("{\"status\":\"ok\"}")] + [InlineData("{\"status\":\"ok\",\"hasDefaultCredentials\":false}")] + [InlineData("{\"status\":\"ok\",\"ssoEnabled\":true}")] + [InlineData("{\"status\":\"ok\",\"hasDefaultCredentials\":false,\"ssoEnabled\":\"true\"}")] + [InlineData("not-json")] + public async Task DetectThirdPartyDnsAsync_NonTechnitiumStatusResponse_NotDetectedAsTechnitiumDns(string statusResponse) + { + var httpClient = CreateRequestAwareMockHttpClient(request => + { + if (request.RequestUri?.AbsolutePath == "/api/status") + { + return new HttpResponseMessage + { + StatusCode = HttpStatusCode.OK, + Content = new StringContent(statusResponse) + }; + } + + return new HttpResponseMessage { StatusCode = HttpStatusCode.NotFound }; + }); + var detector = CreateDetector(httpClient); + + var result = await detector.DetectThirdPartyDnsAsync(CreateTechnitiumTestNetworks(), customUrl: "http://10.10.20.30:5380"); + + result.Should().ContainSingle(); + result[0].IsTechnitiumDns.Should().BeFalse(); + result[0].DnsProviderName.Should().Be("Third-Party LAN DNS"); + } + + [Fact] + public async Task DetectThirdPartyDnsAsync_TechnitiumStatusUnavailable_FallsBackToHtmlDetection() + { + const string baseUrl = "http://10.10.20.30:5380"; + var requestedPaths = new List(); + var technitiumResponse = @"Technitium DNS ServerTechnitium"; + var httpClient = CreateRequestAwareMockHttpClient(request => + { + requestedPaths.Add(request.RequestUri?.AbsolutePath ?? ""); + + if (request.RequestUri?.GetLeftPart(UriPartial.Authority) == baseUrl && + request.RequestUri.AbsolutePath == "/") + { + return new HttpResponseMessage + { + StatusCode = HttpStatusCode.OK, + Content = new StringContent(technitiumResponse) + }; + } + + return new HttpResponseMessage { StatusCode = HttpStatusCode.NotFound }; + }); + var detector = CreateDetector(httpClient); + + var result = await detector.DetectThirdPartyDnsAsync(CreateTechnitiumTestNetworks(), customUrl: baseUrl); + + requestedPaths.Should().Contain("/api/status"); + result.Should().ContainSingle(); + result[0].IsTechnitiumDns.Should().BeTrue(); + } + [Fact] public async Task DetectThirdPartyDnsAsync_TechnitiumTitleWithoutAssets_NotDetectedAsTechnitiumDns() { @@ -199,6 +286,23 @@ public async Task DetectThirdPartyDnsAsync_TechnitiumTitleWithoutAssets_NotDetec result[0].DnsProviderName.Should().Be("Third-Party LAN DNS"); } + private static List CreateTechnitiumTestNetworks() + { + return + [ + new NetworkInfo + { + Id = "net1", + DhcpEnabled = true, + Name = "Trusted", + VlanId = 1, + Subnet = "10.10.20.0/24", + Gateway = "10.10.20.1", + DnsServers = ["10.10.20.30"] + } + ]; + } + private ThirdPartyDnsDetector CreateDetector(HttpClient? httpClient = null) { httpClient ??= new HttpClient { Timeout = TimeSpan.FromSeconds(3) }; @@ -223,6 +327,20 @@ private static HttpClient CreateMockHttpClient(HttpStatusCode statusCode, string return new HttpClient(handlerMock.Object) { Timeout = TimeSpan.FromSeconds(3) }; } + private static HttpClient CreateRequestAwareMockHttpClient(Func responseFactory) + { + var handlerMock = new Mock(); + handlerMock + .Protected() + .Setup>( + "SendAsync", + ItExpr.IsAny(), + ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => responseFactory(request)); + + return new HttpClient(handlerMock.Object) { Timeout = TimeSpan.FromSeconds(3) }; + } + private static HttpClient CreateTimeoutHttpClient() { var handlerMock = new Mock(); @@ -1457,8 +1575,8 @@ public async Task DetectThirdPartyDnsAsync_SameDnsServerMultipleNetworks_ProbesO // HTTP handler should only be called once per unique IP // Pi-hole probe: 3 attempts (port 80, 443, 8080) // AdGuard Home probe: 3 attempts (port 80, 443, 3000) - // Technitium DNS probe: 4 attempts (port 5380, 53443, 80, 443) - callCount.Should().BeLessThanOrEqualTo(10); + // Technitium DNS probe: 8 attempts (status API and HTML on ports 5380, 53443, 80, 443) + callCount.Should().BeLessThanOrEqualTo(14); } #endregion From 11fbb5051fcd7690454c9a670f99f210d5a24b56 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 10:00:01 -0500 Subject: [PATCH 08/23] ControlD probe: log the resolver's verify.controld.com answer Add a start line and, on the no-match path, log the actual CNAME/A records returned so logs show what the resolver gave back for verify.controld.com (it only carries the ControlD record when the query traverses ControlD). Makes on-gateway ctrld detection diagnosable. --- .../Dns/ThirdPartyDnsDetector.cs | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs b/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs index 4a4ba64df..9ef57a807 100644 --- a/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs +++ b/src/NetworkOptimizer.Audit/Dns/ThirdPartyDnsDetector.cs @@ -514,6 +514,8 @@ private async Task ProbeControlDAsync(string ipAddress, CancellationToken if (!IPAddress.TryParse(ipAddress, out var resolverIp)) return false; + _logger.LogDebug("ControlD probe: querying verify.controld.com via {Resolver}", ipAddress); + try { var lookup = new LookupClient(new LookupClientOptions(resolverIp) @@ -532,24 +534,32 @@ private async Task ProbeControlDAsync(string ipAddress, CancellationToken return false; } - var hasCname = dnsResult.Answers + var cnameTargets = dnsResult.Answers .OfType() - .Any(r => r.CanonicalName.Value - .TrimEnd('.') - .EndsWith("controld.com", StringComparison.OrdinalIgnoreCase)); - - if (!hasCname) - { - var aRecords = dnsResult.Answers - .OfType() - .Select(r => r.Address.ToString()) - .ToList(); + .Select(r => r.CanonicalName.Value.TrimEnd('.')) + .ToList(); + var aRecords = dnsResult.Answers + .OfType() + .Select(r => r.Address.ToString()) + .ToList(); - hasCname = aRecords.Any(ip => ip.StartsWith("147.185.34.")); - } + var hasCname = cnameTargets.Any(t => t.EndsWith("controld.com", StringComparison.OrdinalIgnoreCase)) + || aRecords.Any(ip => ip.StartsWith("147.185.34.")); if (hasCname) + { _logger.LogDebug("ControlD probe: detected via verify.controld.com CNAME through {Resolver}", ipAddress); + } + else + { + // Log the actual answer so a reporter's logs show WHAT the resolver + // returned for verify.controld.com (it only carries the ControlD + // CNAME/A when the query actually traverses ControlD). + _logger.LogDebug("ControlD probe: no ControlD record via {Resolver} (CNAMEs: [{Cnames}], A: [{ARecords}])", + ipAddress, + cnameTargets.Count > 0 ? string.Join(", ", cnameTargets) : "none", + aRecords.Count > 0 ? string.Join(", ", aRecords) : "none"); + } return hasCname; } From 7c265adc3c0493ef21ae2152f4bec9ccfc6a8e71 Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:07:47 -0500 Subject: [PATCH 09/23] Nokia ONT: disable handler cookies (root cause) + raw-socket curl replay fallback (#1029) * Nokia ONT: keep-alive connection fallback for firmware that 401s getUpdateinfo (#929) Some XS-010X-Q firmware binds the auth session to the login TCP connection, so presenting the session cookie on the separate getUpdateinfo socket (forced by the per-request ConnectionClose) is rejected 401 -> /login.html despite a successful login. Add a fallback that retries login + getUpdateinfo over a single shared keep-alive connection (MaxConnectionsPerServer=1) when the fresh-connection path yields no RX reading. Also log the LoginForm Set-Cookie header to reveal whether a non-working unit uses a header cookie with a different name. * Nokia ONT: send browser Referer/Origin/X-Requested-With on GponForm calls (#929) HARs from two testers show the successful getUpdateinfo carries Referer: /moreinfo.html, Origin, and X-Requested-With: XMLHttpRequest, and no cookie at all. A T-Fiber/Metronet CLEI variant gates the call on that Referer and 401s to /login.html without it, while the firmware this was first built from accepted the sessionid cookie. Send both the cookie and the browser headers so a single request shape works for both, no per-firmware branching. Supersedes the keep-alive connection fallback (the connection-binding theory the HARs disproved). Keeps the Set-Cookie diagnostic log. * Nokia ONT: page-walk auth flow for CLEI firmware, cached per config (#929) Confirmed by curl on a T-Fiber/Metronet CLEI XS-010X-Q: getUpdateinfo is gated behind the forced PON-password walk (GET /ponpasswd.html, POST ponpasswd_GetConfig, GET /moreinfo.html) with the session cookie on every step. Add that as a PageWalk flow, tried after the Direct flow and cached per config (ConcurrentDictionary keyed by OntConfiguration.Id, re-detected after restart), same idiom as NetgearCmProvider. Direct stays the bare cookie-only getUpdateinfo that is the only flow confirmed working on real hardware (Liosnel's), tried first so it's never regressed; only a box that fails Direct pays for the walk probe, once, until cached. Browser headers (Referer/Origin/X-Requested-With) and User-Agent apply to login and the PageWalk; Direct's data call is left untouched. * Nokia ONT: gate User-Agent to browser flows, fresh login per flow (#929) Ground-truth check of v2.1.1/v2.2.0: both ship a cookie-only getUpdateinfo (no browser headers), v2.1.1 on a plain client, v2.2.0 adding ConnectionClose. So the Direct data call must carry no User-Agent either - move the UA off the shared client and behind the browserHeaders gate, so it rides login and the PageWalk but not the Direct getUpdateinfo, which now matches the shipped request exactly. Also give each flow its own fresh login: a Direct probe that 401s on the CLEI variant leaves a forced-PON-password session state that could poison a following PageWalk if they shared one login. Matches how the confirmed curl ran (one login, then walk). * Nokia ONT: raw-socket curl replay for firmware that rejects HttpClient framing (#929) The picky CLEI firmware accepts the login+getUpdateinfo sequence from curl and browsers but 401s the logically identical HttpClient requests. Byte captures show the framing delta: HttpClient omits User-Agent and Accept, appends ;charset=utf-8 to Content-Type, orders content headers last, and adds Connection: close - none fully controllable through HttpClient. Replace the PageWalk flow with CurlReplay: a raw TCP replay of the tester-proven curl script, byte-for-byte (headers, order, casing), fresh connection per request. Direct stays first for uncached configs and is reverted to the exact shipped v2.1.1/v2.2.0 bytes (the one flow confirmed working on real hardware). * Nokia ONT: disable handler cookies so the sessionid header survives (#929, thanks @jakerobb) The picky firmware answers login calls with a malformed 'Set-Cookie: Path=/; HttpOnly' header. A default HttpClientHandler's CookieContainer parses 'Path=/' as a literal cookie and then silently substitutes it for the hand-set 'Cookie: sessionid=...' header, so the device never saw the session id - the actual root cause jakerobb isolated and verified on his unit. UseCookies=false makes the Direct flow work on that firmware; the CurlReplay raw-socket fallback stays as defense in depth (immune to handler rewriting by construction). --- .../OntProviders/NokiaXs010xOntProvider.cs | 440 ++++++++++++++++-- .../NokiaXs010xOntProviderTests.cs | 355 ++++++++++++++ 2 files changed, 753 insertions(+), 42 deletions(-) diff --git a/src/NetworkOptimizer.Web/Services/OntProviders/NokiaXs010xOntProvider.cs b/src/NetworkOptimizer.Web/Services/OntProviders/NokiaXs010xOntProvider.cs index 5b3989479..0cf9a9121 100644 --- a/src/NetworkOptimizer.Web/Services/OntProviders/NokiaXs010xOntProvider.cs +++ b/src/NetworkOptimizer.Web/Services/OntProviders/NokiaXs010xOntProvider.cs @@ -1,4 +1,6 @@ +using System.Collections.Concurrent; using System.Globalization; +using System.Net.Sockets; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -22,6 +24,27 @@ namespace NetworkOptimizer.Web.Services.OntProviders; /// -> {"CurrentPonPw","VendorID","VersionID","SerialNum","Mac","ActiveSwVer", /// "StandbySwVer","RxOptPwr"} /// +/// Some firmware accepts this sequence from curl and from a browser but answered 401 -> +/// /login.html when HttpClient sent the logically identical requests. Root cause (found by +/// @jakerobb on #929): those units answer the login calls with a malformed +/// "Set-Cookie: Path=/; HttpOnly" header carrying no name=value pair. A cookie-enabled handler +/// parses "Path=/" as a literal cookie, and once the CookieContainer holds anything for the +/// host it silently replaces the hand-set "Cookie: sessionid=..." header - so the device never +/// saw the session id. curl and raw sockets have no cookie engine and browsers discard the +/// malformed header, which is why every other client worked. +/// therefore disables handler cookies so the hand-set header goes out untouched. +/// +/// Firmware behavior demonstrably varies across units, so two flows are kept and the winner is +/// cached per config (see ). is the plain +/// HttpClient sequence as shipped in v2.1.1/v2.2.0 (confirmed working on @Liosnel's unit) plus +/// the cookie fix, which only changes the wire when a device emits Set-Cookie on login. +/// is defense in depth: it writes the raw HTTP requests of +/// the tester-proven curl script over a plain TCP socket, byte-for-byte (same headers, order +/// and casing; fresh connection per request like separate curl invocations), including the +/// forced PON-password page walk some firmware presents: GET /login.html, login, +/// GET /ponpasswd.html, POST /GponForm/ponpasswd_GetConfig, GET /moreinfo.html, then +/// getUpdateinfo. It is immune to any handler-level rewriting by construction. +/// /// The device exposes no TX power, temperature, or explicit link state; the receive /// optical power (RxOptPwr, dBm) is the one health metric it reports. Login credentials /// default to admin/1234 on these units but are user-configurable. @@ -35,10 +58,38 @@ public sealed class NokiaXs010xOntProvider : IOntProvider private const string LoginConfigPath = "/GponForm/Login_GetConfig"; private const string LoginPath = "/GponForm/LoginForm"; private const string UpdateInfoPath = "/GponForm/getUpdateinfo"; + private const string PonPasswdConfigPath = "/GponForm/ponpasswd_GetConfig"; private const string FormContentType = "application/x-www-form-urlencoded"; + private const string LoginPagePath = "/login.html"; + private const string PonPasswdPagePath = "/ponpasswd.html"; + private const string MoreInfoPagePath = "/moreinfo.html"; + private const string BrowserUserAgent = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"; + + /// + /// How a given firmware's getUpdateinfo is reached. Determined on first contact and + /// cached per config in ; re-detected after a process restart. + /// + private enum AuthFlow + { + /// Plain HttpClient login + cookie-only getUpdateinfo (handler cookies off so + /// the hand-set session header survives malformed device Set-Cookie headers). + Direct, + + /// Raw-socket byte-for-byte replay of the tester-proven curl script, immune to + /// handler-level rewriting by construction. + CurlReplay, + } private readonly ILogger _logger; + /// + /// Remembers the that last succeeded per OntConfiguration.Id so + /// later polls go straight to it instead of re-probing the wrong flow (and, for the curl + /// replay, its extra requests) every interval. Re-detected after a restart. + /// + private readonly ConcurrentDictionary _flowCache = new(); + public NokiaXs010xOntProvider(ILogger logger) { _logger = logger; @@ -56,38 +107,7 @@ public NokiaXs010xOntProvider(ILogger logger) { using var client = CreateClient(); var baseUrl = BuildBaseUrl(context); - - OntStats? stats = null; - const int maxAttempts = 3; - for (var attempt = 1; attempt <= maxAttempts; attempt++) - { - var cookieId = await LoginAsync(client, baseUrl, context, cancellationToken); - if (cookieId is not null) - { - var infoJson = await GetUpdateInfoAsync(client, baseUrl, cookieId, context.Name, cancellationToken); - stats = new OntStats - { - Timestamp = DateTime.UtcNow, - DeviceHost = context.ConfiguredHost ?? context.Host, - DeviceName = context.Name, - DeviceModel = "Nokia XS-010X-Q", - }; - ApplyUpdateInfo(infoJson, stats); - if (stats.RxPowerDbm is not null) - break; - } - - // The device sometimes returns an unauthenticated/empty response when the login - // and data request race on the same client - the browser flow and the working - // curl script both hit fresh connections with pauses between steps. A fresh login - // on a fresh connection (ConnectionClose in CreateClient) usually settles it. - if (attempt < maxAttempts) - { - _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: no RX power on attempt {Attempt}/{Max}, retrying", - context.Name, attempt, maxAttempts); - await Task.Delay(TimeSpan.FromMilliseconds(600), cancellationToken); - } - } + var stats = await FetchStatsAsync(client, baseUrl, context, cancellationToken); if (stats is null) { @@ -121,16 +141,11 @@ public NokiaXs010xOntProvider(ILogger logger) { using var client = CreateClient(); var baseUrl = BuildBaseUrl(context); + var stats = await FetchStatsAsync(client, baseUrl, context, cancellationToken); - var cookieId = await LoginAsync(client, baseUrl, context, cancellationToken); - if (cookieId is null) + if (stats is null) return (false, "Login failed - check username/password (default is admin/1234)"); - var infoJson = await GetUpdateInfoAsync(client, baseUrl, cookieId, context.Name, cancellationToken); - - var stats = new OntStats(); - ApplyUpdateInfo(infoJson, stats); - if (stats.RxPowerDbm is null) return (false, "Logged in but response did not contain the expected RxOptPwr field"); @@ -142,6 +157,74 @@ public NokiaXs010xOntProvider(ILogger logger) } } + /// + /// Reads getUpdateinfo trying the resolved order, returning the first + /// result that carries an RX reading and caching the flow that produced it. Each flow does its + /// own login so a probe of the wrong flow (e.g. a Direct getUpdateinfo that 401s on the CLEI + /// variant, which forces a PON-password session state) can't poison the next flow's session. + /// Returns the last built stats (which may lack RX) if a flow logged in but returned no data, + /// or null if every login failed. + /// + private async Task FetchStatsAsync( + HttpClient client, string baseUrl, OntPollContext context, CancellationToken ct) + { + OntStats? last = null; + foreach (var flow in ResolveFlows(context)) + { + string? infoJson; + if (flow == AuthFlow.CurlReplay) + { + infoJson = await GetUpdateInfoViaCurlReplayAsync(baseUrl, context, ct); + } + else + { + var cookieId = await LoginAsync(client, baseUrl, context, ct); + infoJson = cookieId is null + ? null + : await GetUpdateInfoAsync(client, baseUrl, cookieId, context.Name, ct); + } + + if (infoJson is null) + continue; + + var stats = new OntStats + { + Timestamp = DateTime.UtcNow, + DeviceHost = context.ConfiguredHost ?? context.Host, + DeviceName = context.Name, + DeviceModel = "Nokia XS-010X-Q", + }; + ApplyUpdateInfo(infoJson, stats); + if (stats.RxPowerDbm is not null) + { + if (context.Id > 0) + _flowCache[context.Id] = flow; + return stats; + } + + last = stats; + } + + return last; + } + + /// + /// The getUpdateinfo flows to try, in order: the one cached for this config first (so a + /// known-good firmware never re-probes), then the other as a fallback in case the cache is + /// empty or the firmware behavior changed. Uncached configs try Direct first - it's two + /// requests and the only flow confirmed working on real hardware (@Liosnel's), so only a + /// variant that needs the replay pays for the extra probe, once, until it's cached. + /// + private IReadOnlyList ResolveFlows(OntPollContext context) + { + if (context.Id > 0 && _flowCache.TryGetValue(context.Id, out var cached)) + return cached == AuthFlow.CurlReplay + ? new[] { AuthFlow.CurlReplay, AuthFlow.Direct } + : new[] { AuthFlow.Direct, AuthFlow.CurlReplay }; + + return new[] { AuthFlow.Direct, AuthFlow.CurlReplay }; + } + /// /// Runs the three-step GponForm login and returns the session cookie id, or null if /// authentication fails. The cookie is delivered inside the LoginForm JSON body (the @@ -174,16 +257,23 @@ public NokiaXs010xOntProvider(ILogger logger) string loginJson; int loginStatus; + string? setCookie; using (var content = new StringContent(body, Encoding.UTF8, FormContentType)) using (var response = await client.PostAsync($"{baseUrl}{LoginPath}", content, ct)) { loginStatus = (int)response.StatusCode; loginJson = await response.Content.ReadAsStringAsync(ct); + // Diagnostic: the reference flow delivers the cookie in the JSON body and clears the + // Set-Cookie header, but some firmware may set a real (differently named) cookie via + // header. Log it so a non-working unit's logs reveal which path it uses. + setCookie = response.Headers.TryGetValues("Set-Cookie", out var cookieValues) + ? string.Join(" | ", cookieValues) + : null; } var cookieId = ParseCookieId(loginJson); - _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: LoginForm HTTP {Status}, gotCookie={HasCookie}, body={Body}", - context.Name, loginStatus, cookieId != null, Preview(loginJson)); + _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: LoginForm HTTP {Status}, gotCookie={HasCookie}, setCookie={SetCookie}, body={Body}", + context.Name, loginStatus, cookieId != null, setCookie ?? "(none)", Preview(loginJson)); return cookieId; } @@ -203,6 +293,265 @@ private async Task GetUpdateInfoAsync( return body; } + /// + /// Replays the tester-proven curl sequence over raw sockets: GET /login.html, the two login + /// POSTs, the PON-password page walk, then getUpdateinfo, every request framed byte-for-byte + /// like curl's. Walk-page failures are ignored (firmware without the forced page just 404s + /// them); returns the getUpdateinfo body, or null when login fails or the device is + /// unreachable. + /// + private async Task GetUpdateInfoViaCurlReplayAsync( + string baseUrl, OntPollContext context, CancellationToken ct) + { + var username = string.IsNullOrWhiteSpace(context.Username) ? "admin" : context.Username; + var password = context.Password ?? ""; + + await SendCurlRequestAsync(baseUrl, context, LoginPagePath, cookieId: null, refererPath: null, formBody: null, ct); + + var config = await SendCurlRequestAsync(baseUrl, context, LoginConfigPath, cookieId: null, LoginPagePath, "token=token", ct); + if (config is null) + return null; + + var (nonce, saltval) = ParseLoginConfig(config.Value.Body); + if (string.IsNullOrEmpty(nonce)) + { + _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: curl-replay Login_GetConfig HTTP {Status} returned no nonce, body={Body}", + context.Name, config.Value.Status, Preview(config.Value.Body)); + return null; + } + + var cmt = ComputeCmt(username, saltval ?? "", password); + var loginBody = $"cmt={cmt}&nonce={Uri.EscapeDataString(nonce)}"; + var login = await SendCurlRequestAsync(baseUrl, context, LoginPath, cookieId: null, LoginPagePath, loginBody, ct); + if (login is null) + return null; + + var cookieId = ParseCookieId(login.Value.Body); + _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: curl-replay LoginForm HTTP {Status}, gotCookie={HasCookie}, body={Body}", + context.Name, login.Value.Status, cookieId != null, Preview(login.Value.Body)); + if (cookieId is null) + return null; + + await SendCurlRequestAsync(baseUrl, context, PonPasswdPagePath, cookieId, LoginPagePath, formBody: null, ct); + await SendCurlRequestAsync(baseUrl, context, PonPasswdConfigPath, cookieId, PonPasswdPagePath, "token=token", ct); + await SendCurlRequestAsync(baseUrl, context, MoreInfoPagePath, cookieId, PonPasswdPagePath, formBody: null, ct); + + var info = await SendCurlRequestAsync(baseUrl, context, UpdateInfoPath, cookieId, MoreInfoPagePath, "token=token", ct); + if (info is null) + return null; + + _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: curl-replay getUpdateinfo HTTP {Status}, body={Body}", + context.Name, info.Value.Status, Preview(info.Value.Body)); + return info.Value.Body; + } + + /// + /// Sends one curl-framed request on its own TCP connection (each curl invocation in the + /// reference script is a separate process and connection) and reads the response until the + /// device closes the connection or the body is provably complete. Returns null on network + /// errors or timeout so the caller can treat the step as failed without aborting the poll. + /// + private async Task<(int Status, string Body)?> SendCurlRequestAsync( + string baseUrl, OntPollContext context, string path, string? cookieId, string? refererPath, + string? formBody, CancellationToken ct) + { + var requestBytes = BuildCurlRequest(baseUrl, path, cookieId, refererPath, formBody); + var port = context.Port > 0 ? context.Port : 80; + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(TimeoutSeconds)); + var token = timeoutCts.Token; + + try + { + using var tcp = new TcpClient(); + await tcp.ConnectAsync(context.Host, port, token); + var stream = tcp.GetStream(); + await stream.WriteAsync(requestBytes, token); + + using var memory = new MemoryStream(); + var buffer = new byte[8192]; + while (true) + { + var read = await stream.ReadAsync(buffer, token); + if (read == 0) + break; + memory.Write(buffer, 0, read); + if (IsRawResponseComplete(new ReadOnlySpan(memory.GetBuffer(), 0, (int)memory.Length))) + break; + } + + return ParseRawHttpResponse(memory.ToArray()); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + _logger.LogDebug("Nokia XS-010X-Q ONT {Name}: curl-replay request to {Path} timed out", context.Name, path); + return null; + } + catch (Exception ex) when (ex is IOException or SocketException) + { + _logger.LogDebug(ex, "Nokia XS-010X-Q ONT {Name}: curl-replay request to {Path} failed", context.Name, path); + return null; + } + } + + /// + /// Builds one request of the curl replay, byte-for-byte as curl frames it (captured against a + /// local listener from the tester's script): request line, Host, User-Agent, Accept: */*, + /// then Cookie, then Referer, and for POSTs Origin, X-Requested-With, Content-Length before + /// Content-Type with no charset suffix. No Connection header (curl relies on HTTP/1.1 + /// keep-alive; the device closes the connection itself). A null + /// makes it a GET page navigation, which carries none of the POST-only headers. + /// + internal static byte[] BuildCurlRequest( + string baseUrl, string path, string? cookieId, string? refererPath, string? formBody) + { + var host = baseUrl["http://".Length..]; + var builder = new StringBuilder() + .Append(formBody is null ? "GET " : "POST ").Append(path).Append(" HTTP/1.1\r\n") + .Append("Host: ").Append(host).Append("\r\n") + .Append("User-Agent: ").Append(BrowserUserAgent).Append("\r\n") + .Append("Accept: */*\r\n"); + + if (cookieId is not null) + builder.Append("Cookie: sessionid=").Append(cookieId).Append("\r\n"); + + if (refererPath is not null) + builder.Append("Referer: ").Append(baseUrl).Append(refererPath).Append("\r\n"); + + if (formBody is not null) + { + builder.Append("Origin: ").Append(baseUrl).Append("\r\n") + .Append("X-Requested-With: XMLHttpRequest\r\n") + .Append("Content-Length: ").Append(Encoding.ASCII.GetByteCount(formBody).ToString(CultureInfo.InvariantCulture)).Append("\r\n") + .Append("Content-Type: ").Append(FormContentType).Append("\r\n"); + } + + builder.Append("\r\n"); + if (formBody is not null) + builder.Append(formBody); + + return Encoding.ASCII.GetBytes(builder.ToString()); + } + + /// + /// Whether a raw HTTP response buffered so far is provably complete: headers received and the + /// chunked body's terminating zero chunk seen, or the declared Content-Length satisfied. A + /// response with neither framing can only be completed by the device closing the connection, + /// so it reports false and the reader waits for EOF. The device answers GponForm calls with + /// Transfer-Encoding: chunked and Connection: close, making this an early-out; the EOF path + /// is the safety net. + /// + internal static bool IsRawResponseComplete(ReadOnlySpan data) + { + var headerEnd = data.IndexOf("\r\n\r\n"u8); + if (headerEnd < 0) + return false; + + var headers = Encoding.ASCII.GetString(data[..headerEnd]); + var body = data[(headerEnd + 4)..]; + + if (headers.Contains("Transfer-Encoding: chunked", StringComparison.OrdinalIgnoreCase)) + return TryDecodeChunked(body, out _); + + var contentLength = ParseContentLength(headers); + return contentLength is not null && body.Length >= contentLength.Value; + } + + /// + /// Splits a raw HTTP/1.1 response into status code and decoded body text, de-chunking when + /// the device uses Transfer-Encoding: chunked (it does, on every GponForm response) and + /// otherwise honoring Content-Length or taking everything after the headers. Best-effort on + /// truncated input: whatever body bytes arrived are returned. + /// + internal static (int StatusCode, string Body) ParseRawHttpResponse(byte[] raw) + { + var data = raw.AsSpan(); + var headerEnd = data.IndexOf("\r\n\r\n"u8); + if (headerEnd < 0) + return (0, ""); + + var headers = Encoding.ASCII.GetString(data[..headerEnd]); + var body = data[(headerEnd + 4)..]; + + var statusCode = 0; + var statusLine = headers.Split("\r\n", 2)[0].Split(' '); + if (statusLine.Length >= 2) + int.TryParse(statusLine[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out statusCode); + + if (headers.Contains("Transfer-Encoding: chunked", StringComparison.OrdinalIgnoreCase)) + { + TryDecodeChunked(body, out var decoded); + return (statusCode, Encoding.UTF8.GetString(decoded)); + } + + var contentLength = ParseContentLength(headers); + if (contentLength is not null && body.Length > contentLength.Value) + body = body[..contentLength.Value]; + + return (statusCode, Encoding.UTF8.GetString(body)); + } + + /// + /// Walks a chunked transfer-coded body, collecting the chunk payloads received so far into + /// . Returns true only when the terminating zero-size chunk has + /// arrived; on truncated or malformed input it returns false with the chunks recovered up to + /// that point. + /// + private static bool TryDecodeChunked(ReadOnlySpan body, out byte[] decoded) + { + using var output = new MemoryStream(); + var pos = 0; + while (true) + { + var lineEnd = body[pos..].IndexOf("\r\n"u8); + if (lineEnd < 0) + break; + + var sizeText = Encoding.ASCII.GetString(body.Slice(pos, lineEnd)); + var semicolon = sizeText.IndexOf(';'); + if (semicolon >= 0) + sizeText = sizeText[..semicolon]; + + if (!int.TryParse(sizeText.Trim(), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var size) || size < 0) + break; + + pos += lineEnd + 2; + if (size == 0) + { + decoded = output.ToArray(); + return true; + } + + if (pos + size > body.Length) + { + output.Write(body[pos..]); + break; + } + + output.Write(body.Slice(pos, size)); + pos += size + 2; + if (pos > body.Length) + break; + } + + decoded = output.ToArray(); + return false; + } + + /// Reads the Content-Length header value out of a raw response's header block. + private static int? ParseContentLength(string headers) + { + foreach (var line in headers.Split("\r\n")) + { + if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase) && + int.TryParse(line["Content-Length:".Length..].Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var length)) + return length; + } + + return null; + } + /// Trims a raw device response for diagnostic logging. private static string Preview(string s) => string.IsNullOrEmpty(s) ? "(empty)" : (s.Length > 800 ? s[..800] + "...(truncated)" : s); @@ -311,7 +660,14 @@ internal static void ApplyUpdateInfo(string json, OntStats stats) internal static HttpClient CreateClient() { - var client = new HttpClient { Timeout = TimeSpan.FromSeconds(TimeoutSeconds) }; + // Handler cookies must stay off (thanks @jakerobb, #929): some firmware answers the + // login calls with a malformed "Set-Cookie: Path=/; HttpOnly" header, which a default + // CookieContainer parses as a literal cookie and then silently substitutes for the + // hand-set "Cookie: sessionid=..." header, so the device never sees the session id. + // The real session cookie arrives in the LoginForm JSON body, not in Set-Cookie, so + // nothing legitimate is lost by disabling the cookie engine. + var handler = new HttpClientHandler { UseCookies = false }; + var client = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(TimeoutSeconds) }; // Mirror the working curl flow: a fresh TCP connection per request. These GponForm // boxes can tie the login session to the connection, so keep-alive reuse across the // login -> getUpdateinfo steps can return an empty/unauthenticated response. diff --git a/tests/NetworkOptimizer.Web.Tests/NokiaXs010xOntProviderTests.cs b/tests/NetworkOptimizer.Web.Tests/NokiaXs010xOntProviderTests.cs index 59026bcf4..0128cd64b 100644 --- a/tests/NetworkOptimizer.Web.Tests/NokiaXs010xOntProviderTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/NokiaXs010xOntProviderTests.cs @@ -1,5 +1,10 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; using NetworkOptimizer.Monitoring.Models; +using NetworkOptimizer.Monitoring.Providers; using NetworkOptimizer.Web.Services.OntProviders; using Xunit; @@ -95,6 +100,356 @@ public void ParseLoginConfig_MalformedJson_ReturnsNulls() salt.Should().BeNull(); } + // The exact User-Agent the tester-proven curl script sends (also the provider's constant). + private const string CurlScriptUserAgent = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36"; + + [Fact] + public void BuildCurlRequest_GetUpdateinfo_MatchesCapturedCurlFraming() + { + // Byte-for-byte template captured by replaying the tester's working curl script against + // a local listener: Host, User-Agent, Accept, Cookie, Referer, Origin, X-Requested-With, + // then Content-Length BEFORE Content-Type with no charset suffix, and no Connection header. + var expected = + "POST /GponForm/getUpdateinfo HTTP/1.1\r\n" + + "Host: 192.0.2.1\r\n" + + $"User-Agent: {CurlScriptUserAgent}\r\n" + + "Accept: */*\r\n" + + "Cookie: sessionid=deadbeefcafe0123\r\n" + + "Referer: http://192.0.2.1/moreinfo.html\r\n" + + "Origin: http://192.0.2.1\r\n" + + "X-Requested-With: XMLHttpRequest\r\n" + + "Content-Length: 11\r\n" + + "Content-Type: application/x-www-form-urlencoded\r\n" + + "\r\n" + + "token=token"; + + var bytes = NokiaXs010xOntProvider.BuildCurlRequest( + "http://192.0.2.1", "/GponForm/getUpdateinfo", "deadbeefcafe0123", "/moreinfo.html", "token=token"); + + System.Text.Encoding.ASCII.GetString(bytes).Should().Be(expected); + } + + [Fact] + public void BuildCurlRequest_LoginConfigWithoutCookie_OmitsCookieHeader() + { + var expected = + "POST /GponForm/Login_GetConfig HTTP/1.1\r\n" + + "Host: 192.0.2.1:8080\r\n" + + $"User-Agent: {CurlScriptUserAgent}\r\n" + + "Accept: */*\r\n" + + "Referer: http://192.0.2.1:8080/login.html\r\n" + + "Origin: http://192.0.2.1:8080\r\n" + + "X-Requested-With: XMLHttpRequest\r\n" + + "Content-Length: 11\r\n" + + "Content-Type: application/x-www-form-urlencoded\r\n" + + "\r\n" + + "token=token"; + + var bytes = NokiaXs010xOntProvider.BuildCurlRequest( + "http://192.0.2.1:8080", "/GponForm/Login_GetConfig", cookieId: null, "/login.html", "token=token"); + + System.Text.Encoding.ASCII.GetString(bytes).Should().Be(expected); + } + + [Fact] + public void BuildCurlRequest_PageGet_CarriesOnlyCookieAndReferer() + { + var expected = + "GET /ponpasswd.html HTTP/1.1\r\n" + + "Host: 192.0.2.1\r\n" + + $"User-Agent: {CurlScriptUserAgent}\r\n" + + "Accept: */*\r\n" + + "Cookie: sessionid=deadbeefcafe0123\r\n" + + "Referer: http://192.0.2.1/login.html\r\n" + + "\r\n"; + + var bytes = NokiaXs010xOntProvider.BuildCurlRequest( + "http://192.0.2.1", "/ponpasswd.html", "deadbeefcafe0123", "/login.html", formBody: null); + + System.Text.Encoding.ASCII.GetString(bytes).Should().Be(expected); + } + + [Fact] + public void BuildCurlRequest_InitialLoginPageGet_HasNoCookieOrReferer() + { + var expected = + "GET /login.html HTTP/1.1\r\n" + + "Host: 192.0.2.1\r\n" + + $"User-Agent: {CurlScriptUserAgent}\r\n" + + "Accept: */*\r\n" + + "\r\n"; + + var bytes = NokiaXs010xOntProvider.BuildCurlRequest( + "http://192.0.2.1", "/login.html", cookieId: null, refererPath: null, formBody: null); + + System.Text.Encoding.ASCII.GetString(bytes).Should().Be(expected); + } + + // The device answers every GponForm call with Transfer-Encoding: chunked + Connection: close + // (seen in both testers' HARs), so the raw reader has to de-chunk. + private const string ChunkedDeviceResponse = + "HTTP/1.1 200 OK\r\nServer: nginx\r\nContent-Type: text/html\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + + "8\r\n{\"RxOptP\r\nc\r\nwr\":\"-13.3\"}\r\n0\r\n\r\n"; + + [Fact] + public void ParseRawHttpResponse_ChunkedBody_DecodesStatusAndBody() + { + var (status, body) = NokiaXs010xOntProvider.ParseRawHttpResponse( + System.Text.Encoding.ASCII.GetBytes(ChunkedDeviceResponse)); + + status.Should().Be(200); + body.Should().Be("""{"RxOptPwr":"-13.3"}"""); + } + + [Fact] + public void ParseRawHttpResponse_ContentLengthBody_TrimsToDeclaredLength() + { + var raw = System.Text.Encoding.ASCII.GetBytes( + "HTTP/1.1 401 Unauthorized\r\nContent-Length: 5\r\n\r\nhelloEXTRA"); + + var (status, body) = NokiaXs010xOntProvider.ParseRawHttpResponse(raw); + + status.Should().Be(401); + body.Should().Be("hello"); + } + + [Fact] + public void ParseRawHttpResponse_TruncatedChunkedBody_ReturnsBytesReceivedSoFar() + { + var raw = System.Text.Encoding.ASCII.GetBytes( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n8\r\n{\"RxOptP"); + + var (status, body) = NokiaXs010xOntProvider.ParseRawHttpResponse(raw); + + status.Should().Be(200); + body.Should().Be("{\"RxOptP"); + } + + [Fact] + public void IsRawResponseComplete_ChunkedWithTerminator_IsComplete() + { + NokiaXs010xOntProvider.IsRawResponseComplete( + System.Text.Encoding.ASCII.GetBytes(ChunkedDeviceResponse)).Should().BeTrue(); + } + + [Fact] + public void IsRawResponseComplete_ChunkedWithoutTerminator_IsIncomplete() + { + var raw = System.Text.Encoding.ASCII.GetBytes( + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n8\r\n{\"RxOptP\r\n"); + + NokiaXs010xOntProvider.IsRawResponseComplete(raw).Should().BeFalse(); + } + + [Fact] + public void IsRawResponseComplete_NoLengthFraming_WaitsForConnectionClose() + { + var raw = System.Text.Encoding.ASCII.GetBytes( + "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\nbody"); + + NokiaXs010xOntProvider.IsRawResponseComplete(raw).Should().BeFalse(); + } + + [Fact] + public async Task PollAsync_UserAgentGatedFirmware_FallsBackToCurlReplay() + { + // Simulates the picky firmware: login works for any client, but getUpdateinfo 401s + // unless the request carries a User-Agent (the trait every proven-working client - curl + // and browsers - shares and the shipped HttpClient request lacks). The Direct flow must + // probe first and fail, then the raw-socket curl replay must complete the walk and read. + await using var server = new PickyGponFormServer(); + var provider = new NokiaXs010xOntProvider(NullLogger.Instance); + var context = new OntPollContext + { + Id = 1, + Name = "TestOnt", + Host = "127.0.0.1", + Port = server.Port, + Username = "admin", + Password = "1234", + }; + + var stats = await provider.PollAsync(context); + + stats.Should().NotBeNull(); + stats!.RxPowerDbm.Should().BeApproximately(-13.3, 0.0001); + stats.VendorSn.Should().Be("ALCLaabbccdd"); + server.RejectedUserAgentlessUpdateInfo.Should().BeTrue("the Direct flow should have probed first and been 401'd"); + } + + [Fact] + public async Task PollAsync_MalformedSetCookieFirmware_DirectFlowStillAuthenticates() + { + // Simulates the root cause @jakerobb isolated on #929: the firmware answers the login + // calls with a malformed "Set-Cookie: Path=/; HttpOnly" header (no name=value). With a + // cookie-enabled handler the CookieContainer swallows it and substitutes garbage for the + // hand-set sessionid header; with UseCookies off the Direct flow must succeed as-is, + // without ever falling back to the curl replay. + await using var server = new PickyGponFormServer(requireUserAgent: false, emitMalformedSetCookie: true); + var provider = new NokiaXs010xOntProvider(NullLogger.Instance); + var context = new OntPollContext + { + Id = 2, + Name = "TestOnt", + Host = "127.0.0.1", + Port = server.Port, + Username = "admin", + Password = "1234", + }; + + var stats = await provider.PollAsync(context); + + stats.Should().NotBeNull(); + stats!.RxPowerDbm.Should().BeApproximately(-13.3, 0.0001); + server.SawWalkPage.Should().BeFalse("Direct should authenticate without the curl-replay fallback"); + } + + /// + /// Minimal GponForm device double that mimics the picky firmware variants: serves the login + /// handshake to anyone, but 401s getUpdateinfo when the session cookie is wrong (or, when + /// requireUserAgent is set, when the request has no User-Agent header), and can emit + /// the malformed Set-Cookie header the real units send on login responses. One request per + /// connection, Connection: close, like the device. + /// + private sealed class PickyGponFormServer : IAsyncDisposable + { + private const string Nonce = "AbCdEfGhIjKlMnOpQrStUvWxYz012345"; + private const string Salt = "ea"; + private const string CookieId = "deadbeefcafe0123"; + + private readonly TcpListener _listener; + private readonly CancellationTokenSource _cts = new(); + private readonly Task _acceptLoop; + private readonly bool _requireUserAgent; + private readonly bool _emitMalformedSetCookie; + + public int Port { get; } + + public bool RejectedUserAgentlessUpdateInfo { get; private set; } + + public bool SawWalkPage { get; private set; } + + public PickyGponFormServer(bool requireUserAgent = true, bool emitMalformedSetCookie = false) + { + _requireUserAgent = requireUserAgent; + _emitMalformedSetCookie = emitMalformedSetCookie; + _listener = new TcpListener(IPAddress.Loopback, 0); + _listener.Start(); + Port = ((IPEndPoint)_listener.LocalEndpoint).Port; + _acceptLoop = AcceptLoopAsync(); + } + + private async Task AcceptLoopAsync() + { + try + { + while (!_cts.IsCancellationRequested) + { + using var client = await _listener.AcceptTcpClientAsync(_cts.Token); + await HandleAsync(client); + } + } + catch (OperationCanceledException) { } + catch (SocketException) { } + } + + private async Task HandleAsync(TcpClient client) + { + var stream = client.GetStream(); + var buffer = new byte[16384]; + var received = new MemoryStream(); + string request; + int headerEnd; + while (true) + { + var read = await stream.ReadAsync(buffer, _cts.Token); + if (read == 0) + return; + received.Write(buffer, 0, read); + request = Encoding.ASCII.GetString(received.ToArray()); + headerEnd = request.IndexOf("\r\n\r\n", StringComparison.Ordinal); + if (headerEnd < 0) + continue; + if (request.Length >= headerEnd + 4 + GetContentLength(request[..headerEnd])) + break; + } + + var lines = request[..headerEnd].Split("\r\n"); + var target = lines[0].Split(' ')[1]; + var body = request[(headerEnd + 4)..]; + var hasUserAgent = lines.Any(l => l.StartsWith("User-Agent:", StringComparison.OrdinalIgnoreCase)); + var hasSession = lines.Any(l => + l.StartsWith("Cookie:", StringComparison.OrdinalIgnoreCase) && l.Contains($"sessionid={CookieId}")); + + if (target == "/ponpasswd.html") + SawWalkPage = true; + + var status = "200 OK"; + var setCookie = ""; + string responseBody; + switch (target) + { + case "/GponForm/Login_GetConfig": + responseBody = $$"""{"XError":0,"nonce":"{{Nonce}}","saltval":"{{Salt}}"}"""; + if (_emitMalformedSetCookie) + setCookie = "Set-Cookie: Path=/; HttpOnly\r\n"; + break; + case "/GponForm/LoginForm": + var expectedCmt = NokiaXs010xOntProvider.ComputeCmt("admin", Salt, "1234"); + responseBody = body.Contains($"cmt={expectedCmt}") && body.Contains($"nonce={Nonce}") + ? $$"""{"login_result":"success","cookieid":"{{CookieId}}"}""" + : """{"login_result":"error"}"""; + if (_emitMalformedSetCookie) + setCookie = "Set-Cookie: Path=/; HttpOnly\r\n"; + break; + case "/GponForm/getUpdateinfo": + if ((hasUserAgent || !_requireUserAgent) && hasSession) + { + responseBody = FullUpdateInfoJson; + } + else + { + if (!hasUserAgent) + RejectedUserAgentlessUpdateInfo = true; + status = "401 Unauthorized"; + responseBody = "login"; + } + break; + default: + responseBody = ""; + break; + } + + var response = + $"HTTP/1.1 {status}\r\nContent-Type: text/html\r\n{setCookie}" + + $"Content-Length: {Encoding.ASCII.GetByteCount(responseBody)}\r\nConnection: close\r\n\r\n" + + responseBody; + await stream.WriteAsync(Encoding.ASCII.GetBytes(response), _cts.Token); + } + + private static int GetContentLength(string headers) + { + foreach (var line in headers.Split("\r\n")) + { + if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase) && + int.TryParse(line["Content-Length:".Length..].Trim(), out var length)) + return length; + } + + return 0; + } + + public async ValueTask DisposeAsync() + { + _cts.Cancel(); + _listener.Stop(); + try { await _acceptLoop; } catch { } + _cts.Dispose(); + } + } + [Fact] public void ParseCookieId_SuccessResponse_ReturnsCookie() { From e8bbb50824aacd8626fd6c4bf02ef12991e83a74 Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:05:08 -0500 Subject: [PATCH 10/23] Multi-Site: stop reverse proxies silently breaking the agent tunnel, and report agent(s) offline when it's down (#1032) * Agent status: don't report online off REST heartbeats when the tunnel is down When the server offers an agent tunnel, an agent that only ever REST-heartbeats has a dead or unpublished tunnel (e.g. the gRPC tunnel path isn't reverse-proxied) with every tunnel-dependent feature down. Heartbeat freshness alone no longer counts as online in that case - only an open tunnel, or a short reconnect grace after a previously-connected one drops. When the server offers no tunnel at all, REST heartbeat freshness still means online (single-box/legacy deployments). LAN speed-test target resolution keeps the looser reachability check (IsReachableForLanTest): those tests hit the agent's nginx directly, not the tunnel, so a heartbeat-only agent is still a valid target. * Review: keep the client speed-test placeholder on the LAN reachability check The Client Speed Test target placeholder is documented to mirror SiteSpeedTestTargetResolver's pick, which resolves via GetOnlineAgentLanIpAsync (IsReachableForLanTest). Splitting IsAgentLive left the placeholder on the stricter status check, so a heartbeat-only agent would show the generic example instead of the real auto-resolved LAN target. Point it at the same reachability predicate and refresh a now-stale comment. * Rename IsReachableForLanTest -> IsReachable The predicate only checks reachability (open tunnel or fresh heartbeat); it does not verify the agent actually hosts a LAN speed test (that's an opt-in agent flag the server has no signal for yet - separate TODO). Name it for what it checks and document the limitation, so it no longer implies a capability guarantee it can't make. Pure rename, no behavior change. * Never redirect gRPC from the canonical-host middleware The canonical-host redirect (REVERSE_PROXIED_HOST_NAME) runs on the shared pipeline, so it also sees the agent tunnel's gRPC requests on the HTTP/2 listener. A gRPC stream cannot follow a 302, so when a reverse proxy forwards the tunnel path without presenting the canonical Host (e.g. Caddy proxying to the https://localhost tunnel upstream sends Host: localhost), the Connect stream was redirected to death - while REST heartbeats still landed and kept the agent showing online. Skip the redirect for application/grpc requests: gRPC is machine-to-machine and has no canonical-host concern. --- .../Components/Pages/Settings.razor | 9 +- src/NetworkOptimizer.Web/Program.cs | 17 +++- .../Services/AgentConnectionAlertMonitor.cs | 7 +- .../Services/AgentEnrollmentService.cs | 12 +-- .../Services/AgentTunnelRegistry.cs | 64 ++++++++++++-- .../AgentEnrollmentServiceTests.cs | 2 +- .../AgentTunnelRegistryTests.cs | 84 +++++++++++++++++++ 7 files changed, 173 insertions(+), 22 deletions(-) create mode 100644 tests/NetworkOptimizer.Web.Tests/AgentTunnelRegistryTests.cs diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index 86021888c..4cc6c2660 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -3711,9 +3711,10 @@ } // At-a-glance agent state for a site's row in the Sites table. Uses the - // shared IsAgentLive definition (tunnel-connected or heartbeat-fresh) and is - // kept current by the agent poll timer, so it agrees with the other status - // surfaces instead of freezing at whatever page load saw. + // shared IsAgentLive definition (tunnel-connected, or heartbeat-fresh only + // when this server offers no tunnel) and is kept current by the agent poll + // timer, so it agrees with the other status surfaces instead of freezing at + // whatever page load saw. private (string Dot, string Label, string Tooltip) SiteAgentStatus(int siteId) { var agents = AgentsFor(siteId); @@ -3847,7 +3848,7 @@ if (_agentOnGateway.TryGetValue(site.Id, out var onGateway) && onGateway) return "none - the agent runs on the UniFi gateway; enter a separate box hosting the speed test"; var lanIp = AgentsFor(site.Id) - .Where(a => a.Enabled && !string.IsNullOrEmpty(a.LanIp) && TunnelRegistry.IsAgentLive(a)) + .Where(a => a.Enabled && !string.IsNullOrEmpty(a.LanIp) && TunnelRegistry.IsReachable(a)) .OrderByDescending(a => a.LastSeenAt) .Select(a => a.LanIp) .FirstOrDefault(); diff --git a/src/NetworkOptimizer.Web/Program.cs b/src/NetworkOptimizer.Web/Program.cs index 65c8d3da2..1128b3c62 100644 --- a/src/NetworkOptimizer.Web/Program.cs +++ b/src/NetworkOptimizer.Web/Program.cs @@ -10,14 +10,12 @@ using NetworkOptimizer.Core.Helpers; using NetworkOptimizer.Monitoring.Providers; using NetworkOptimizer.Storage.Models; -using NetworkOptimizer.Storage.Services; using NetworkOptimizer.UniFi; using NetworkOptimizer.Web; using NetworkOptimizer.Web.Endpoints; using NetworkOptimizer.Web.Services; using NetworkOptimizer.Web.Services.CableModemProviders; using NetworkOptimizer.Web.Services.Licensing; -using NetworkOptimizer.Web.Services.CellularModemProviders; using NetworkOptimizer.Web.Services.OntProviders; using NetworkOptimizer.Web.Services.Ssh; using NetworkOptimizer.WiFi.Models; @@ -1060,6 +1058,21 @@ ProductVersion TEXT NOT NULL { app.Use(async (context, next) => { + // The agent tunnel is gRPC, which cannot follow an HTTP redirect: a 302 + // here silently breaks the tunnel whenever a reverse proxy forwards the + // gRPC path without presenting the canonical Host (e.g. Caddy proxying to + // the https://localhost tunnel upstream sends Host: localhost, so the app + // sees a non-canonical host and redirects the Connect stream to death - + // while REST heartbeats still land and keep the agent showing online). + // gRPC is machine-to-machine and has no canonical-host concern, so never + // redirect it; let it through to the gRPC endpoint regardless of Host. + var contentType = context.Request.ContentType; + if (contentType != null && contentType.StartsWith("application/grpc", StringComparison.OrdinalIgnoreCase)) + { + await next(); + return; + } + var requestHost = context.Request.Host.Host; // Check if host matches (case-insensitive) diff --git a/src/NetworkOptimizer.Web/Services/AgentConnectionAlertMonitor.cs b/src/NetworkOptimizer.Web/Services/AgentConnectionAlertMonitor.cs index 3fa35df65..fb0ab6a2b 100644 --- a/src/NetworkOptimizer.Web/Services/AgentConnectionAlertMonitor.cs +++ b/src/NetworkOptimizer.Web/Services/AgentConnectionAlertMonitor.cs @@ -9,9 +9,10 @@ namespace NetworkOptimizer.Web.Services; /// Periodically sweeps enrolled agents and raises alert events when one stays /// offline past the grace threshold, pairing each with a reconnect event when it /// comes back. Sweeps (open tunnel, -/// or heartbeat freshness for REST-only agents and reconnect gaps) instead of -/// hooking tunnel teardown, so agent redeploys and transient tunnel bounces never -/// alert - only an agent continuously offline for the full threshold does. +/// its brief reconnect grace, or - only when this server offers no tunnel - REST +/// heartbeat freshness) instead of hooking tunnel teardown, so agent redeploys +/// and transient tunnel bounces never alert - only an agent continuously offline +/// for the full threshold does. /// public class AgentConnectionAlertMonitor : BackgroundService { diff --git a/src/NetworkOptimizer.Web/Services/AgentEnrollmentService.cs b/src/NetworkOptimizer.Web/Services/AgentEnrollmentService.cs index 40f54f33e..deb79e874 100644 --- a/src/NetworkOptimizer.Web/Services/AgentEnrollmentService.cs +++ b/src/NetworkOptimizer.Web/Services/AgentEnrollmentService.cs @@ -227,17 +227,17 @@ public async Task HeartbeatAsync(string agentKey, string? version, string? if (site == null) return null; - // Filter to live agents FIRST (open tunnel is authoritative and instant, - // heartbeat freshness covers REST-only agents and reconnect gaps - the - // same IsAgentLive definition every status surface uses), then take the - // most recently seen, so a site with one stale and one live agent still - // resolves and the resolved target always matches what the UI shows. + // Filter to reachable agents FIRST (open tunnel, or a fresh REST heartbeat + // - the LAN speed test hits the agent's nginx directly, not the tunnel, so + // a heartbeat-only agent is still a valid target even when its tunnel is + // down), then take the most recently seen, so a site with one stale and one + // reachable agent still resolves. var agents = await db.SiteAgents.AsNoTracking() .Where(a => a.SiteId == site.Id && a.Enabled && a.EnrolledAt != null && a.LanIp != null) .OrderByDescending(a => a.LastSeenAt) .ToListAsync(); - return agents.FirstOrDefault(a => _tunnelRegistry.IsAgentLive(a))?.LanIp; + return agents.FirstOrDefault(a => _tunnelRegistry.IsReachable(a))?.LanIp; } /// Enables or disables an agent. Disabled agents cannot enroll or heartbeat. diff --git a/src/NetworkOptimizer.Web/Services/AgentTunnelRegistry.cs b/src/NetworkOptimizer.Web/Services/AgentTunnelRegistry.cs index d7cdecab3..d90cb7cce 100644 --- a/src/NetworkOptimizer.Web/Services/AgentTunnelRegistry.cs +++ b/src/NetworkOptimizer.Web/Services/AgentTunnelRegistry.cs @@ -15,6 +15,29 @@ public class AgentTunnelRegistry { private readonly ConcurrentDictionary _connections = new(); + // Last time each agent had an open tunnel (stamped on connect and on drop). + // Bridges the brief gap while a previously-connected tunnel reconnects, so a + // real reconnect doesn't flap the online status - without letting a tunnel + // that never connected count as live. + private readonly ConcurrentDictionary _lastTunnelActivity = new(); + + // Whether this server bound the agent tunnel listener. When it did, an agent + // that only ever REST-heartbeats has a dead or unpublished tunnel (e.g. the + // gRPC path isn't reverse-proxied) and its tunnel-dependent features are all + // down, so heartbeat freshness alone must NOT read as online. + private readonly bool _tunnelEnabled; + + // How long after a tunnel drops an agent still counts as live, covering the + // agent's dial-out backoff so a genuine reconnect doesn't blink the status + // offline. Sized just over the agent heartbeat interval (30s) plus a dial + // attempt; a tunnel down longer than this is a real outage, not a reconnect. + private static readonly TimeSpan TunnelReconnectGrace = TimeSpan.FromSeconds(75); + + public AgentTunnelRegistry(AgentTunnelOptions tunnelOptions) + { + _tunnelEnabled = tunnelOptions.Enabled; + } + /// Registers a new live connection, displacing any stale one for the same agent. public AgentTunnelConnection Register(int agentId, string siteSlug, string agentName) { @@ -24,6 +47,7 @@ public AgentTunnelConnection Register(int agentId, string siteSlug, string agent old.Complete(); return connection; }); + _lastTunnelActivity[agentId] = DateTime.UtcNow; return connection; } @@ -34,6 +58,9 @@ public AgentTunnelConnection Register(int agentId, string siteSlug, string agent public void Unregister(AgentTunnelConnection connection) { connection.Complete(); + // Stamp the drop time so the reconnect grace is measured from when the + // tunnel actually went away, not from when it first connected. + _lastTunnelActivity[connection.AgentId] = DateTime.UtcNow; ((ICollection>)_connections) .Remove(new KeyValuePair(connection.AgentId, connection)); } @@ -42,13 +69,38 @@ public void Unregister(AgentTunnelConnection connection) public bool IsConnected(int agentId) => _connections.ContainsKey(agentId); /// - /// Whether an agent counts as online for status displays: an open tunnel is - /// authoritative and instant; otherwise a fresh heartbeat (REST-only agents, - /// or the gap while a tunnel reconnects) keeps it online. The single - /// definition every status surface (site dropdown, All Sites, Multi-Site - /// settings) shares, so they can't disagree on what "online" means. + /// Whether an agent counts as online for status displays. An open tunnel is + /// authoritative and instant. When this server offers no tunnel at all, a + /// fresh REST heartbeat is the best signal and keeps the agent online. But + /// when the server DOES offer a tunnel, a heartbeat-only agent has a dead or + /// unpublished tunnel (e.g. the gRPC path isn't reverse-proxied) with every + /// tunnel-dependent feature down - reporting it online off REST heartbeats + /// alone would be dishonest - so only an open tunnel, or the brief grace + /// while a previously-connected one reconnects, counts. The single definition + /// every status surface (site dropdown, All Sites, Multi-Site settings) + /// shares, so they can't disagree on what "online" means. + /// + public bool IsAgentLive(NetworkOptimizer.Storage.Models.SiteAgent agent) + { + if (IsConnected(agent.Id)) + return true; + if (_tunnelEnabled) + return _lastTunnelActivity.TryGetValue(agent.Id, out var last) + && DateTime.UtcNow - last < TunnelReconnectGrace; + return AgentEnrollmentService.IsOnline(agent.LastSeenAt); + } + + /// + /// Whether we've heard from the agent recently by any means - open tunnel or a + /// fresh REST heartbeat. Deliberately looser than , + /// which requires a working tunnel: this is the reachability signal used to + /// pick a site's LAN speed-test target, since that test hits the agent's own + /// nginx directly rather than the tunnel, so a heartbeat-only agent is still a + /// candidate. It does NOT verify the agent actually hosts a speed test (LAN + /// testing is an opt-in agent flag the server has no signal for yet); the + /// resolver's on-gateway check is the only capability gate today. /// - public bool IsAgentLive(NetworkOptimizer.Storage.Models.SiteAgent agent) => + public bool IsReachable(NetworkOptimizer.Storage.Models.SiteAgent agent) => IsConnected(agent.Id) || AgentEnrollmentService.IsOnline(agent.LastSeenAt); /// Live connections for a site (normally zero or one per agent). diff --git a/tests/NetworkOptimizer.Web.Tests/AgentEnrollmentServiceTests.cs b/tests/NetworkOptimizer.Web.Tests/AgentEnrollmentServiceTests.cs index 217a9b7ad..bc2288ede 100644 --- a/tests/NetworkOptimizer.Web.Tests/AgentEnrollmentServiceTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/AgentEnrollmentServiceTests.cs @@ -18,7 +18,7 @@ private sealed class TestDbFactory : IDbContextFactory + new(new AgentTunnelOptions(Enabled: tunnelEnabled, Port: 0)); + + private static SiteAgent Agent(int id, DateTime? lastSeenAt) => + new() { Id = id, Name = "Agent", EnrolledAt = DateTime.UtcNow, LastSeenAt = lastSeenAt }; + + [Fact] + public void IsAgentLive_TunnelConnected_IsLive() + { + var registry = Registry(tunnelEnabled: true); + var agent = Agent(1, lastSeenAt: DateTime.UtcNow); + registry.Register(agent.Id, "branch-office", "Agent"); + + registry.IsAgentLive(agent).Should().BeTrue(); + } + + [Fact] + public void IsAgentLive_TunnelEnabled_HeartbeatOnly_NeverTunneled_IsOffline() + { + // The reported bug: the gRPC tunnel path isn't reverse-proxied, so the + // agent never tunnels but its REST heartbeat stays fresh. That must NOT + // read as online when the server offers a tunnel. + var registry = Registry(tunnelEnabled: true); + var agent = Agent(1, lastSeenAt: DateTime.UtcNow); + + registry.IsAgentLive(agent).Should().BeFalse(); + } + + [Fact] + public void IsAgentLive_TunnelEnabled_WithinReconnectGraceAfterDrop_StaysLive() + { + // A previously-connected tunnel that just dropped is mid-reconnect, not an + // outage - the grace keeps it live so the status doesn't flap. + var registry = Registry(tunnelEnabled: true); + var agent = Agent(1, lastSeenAt: DateTime.UtcNow); + var connection = registry.Register(agent.Id, "branch-office", "Agent"); + registry.Unregister(connection); + + registry.IsConnected(agent.Id).Should().BeFalse(); + registry.IsAgentLive(agent).Should().BeTrue(); + } + + [Fact] + public void IsAgentLive_TunnelDisabled_HeartbeatFresh_IsLive() + { + // No tunnel listener at all (single-box / legacy deployment): REST + // heartbeat freshness is the best signal and keeps the agent online. + var registry = Registry(tunnelEnabled: false); + var agent = Agent(1, lastSeenAt: DateTime.UtcNow); + + registry.IsAgentLive(agent).Should().BeTrue(); + } + + [Fact] + public void IsAgentLive_TunnelDisabled_HeartbeatStale_IsOffline() + { + var registry = Registry(tunnelEnabled: false); + var agent = Agent(1, lastSeenAt: DateTime.UtcNow - AgentEnrollmentService.OnlineWindow - TimeSpan.FromMinutes(1)); + + registry.IsAgentLive(agent).Should().BeFalse(); + } + + [Fact] + public void IsReachable_HeartbeatFresh_ButBrokenTunnel_IsReachable() + { + // LAN speed tests hit the agent's nginx directly, so a heartbeat-only + // agent (tunnel enabled but never connected) is still reachable - looser + // than IsAgentLive on purpose. + var registry = Registry(tunnelEnabled: true); + var agent = Agent(1, lastSeenAt: DateTime.UtcNow); + + registry.IsAgentLive(agent).Should().BeFalse(); + registry.IsReachable(agent).Should().BeTrue(); + } +} From 76669ff926521e2d180a350c6f48b7522fd0fac9 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 12:31:46 -0500 Subject: [PATCH 11/23] Site setup: lead the ready-screen next steps with connecting the UniFi Console --- .../Components/Shared/SiteSetupWizard.razor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor b/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor index ed3e14370..6185df9db 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor @@ -211,8 +211,8 @@

@_site.Name is ready

The site's database is provisioned@(_connected ? ", its UniFi Console is connected" : "")@(_agentToken != null ? ", and its agent token is issued" : ""). - Switch to the site to configure monitoring, speed tests, and everything else, exactly like a - single-site install. + Switch to the site to @(_connected ? "" : "connect its UniFi Console, then ")configure monitoring, speed tests, + and everything else, exactly like a single-site install.

From 2e11d52aa1dfe3e1c39dd2e659d098b34f5ee8ff Mon Sep 17 00:00:00 2001 From: Optic00 Date: Mon, 20 Jul 2026 18:38:04 +0200 Subject: [PATCH 12/23] Add Zyxel GPON-SFP (PMG3000) ONT provider Read-only telemetry provider for the Zyxel PMG3000 GPON-SFP stick (a MaxLinear/T&W-based unit) over its HTTP CGI API. Basic auth (default admin/1234), GET /cgi/get_gpon_info for optics/link and GET /cgi/get_sn for serial identity. Bodies are JavaScript object literals, not strict JSON, so a small provider-local lenient tokenizer decodes them; no new dependency. Maps line_status via the shared O1-O7 parser plus Rx/Tx power, temperature, voltage and bias current onto OntStats. get_gpon_info is the required endpoint; get_sn is best-effort and never logged (it carries cur_pass). Fixture-driven unit tests against a scrubbed live capture. The existing Realtek provider was ruled out first (endpoints 404, form login vs this device's Basic auth); see the design spec for the evidence. --- .../Components/Pages/Settings.razor | 2 + src/NetworkOptimizer.Web/Program.cs | 1 + .../OntProviders/ZyxelGponSfpOntProvider.cs | 460 ++++++++++++++++++ .../ZyxelGponSfpOntProviderTests.cs | 398 +++++++++++++++ 4 files changed, 861 insertions(+) create mode 100644 src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs create mode 100644 tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index 4ea2b03be..073284b9d 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -1565,6 +1565,7 @@ + @@ -5863,6 +5864,7 @@ "quantum-q1000k" => "Quantum Fiber Q1000K (HTTP)", "telekom-modem2" => "Telekom Glasfaser-Modem 2 (HTTP)", "nokia-xs010x-q" => "Nokia XS-010X-Q (HTTP)", + "zyxel-gpon-sfp" => "Zyxel GPON-SFP (PMG3000, HTTP)", "generic-http-ont" => "Generic HTTP ONT", _ => provider, }; diff --git a/src/NetworkOptimizer.Web/Program.cs b/src/NetworkOptimizer.Web/Program.cs index ed31bdfda..edbf5610b 100644 --- a/src/NetworkOptimizer.Web/Program.cs +++ b/src/NetworkOptimizer.Web/Program.cs @@ -356,6 +356,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddScoped(sp => sp.GetRequiredService() .GetFor(sp.GetRequiredService().Slug).Ont); diff --git a/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs b/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs new file mode 100644 index 000000000..01aadec29 --- /dev/null +++ b/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs @@ -0,0 +1,460 @@ +using System.Globalization; +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging; +using NetworkOptimizer.Monitoring.Models; +using NetworkOptimizer.Monitoring.Providers; + +namespace NetworkOptimizer.Web.Services.OntProviders; + +/// +/// ONT provider for the Zyxel PMG3000 GPON-SFP stick (a MaxLinear/T&W-based unit), +/// used as an in-gateway ONT on GPON FTTH connections. The stick speaks plain HTTP +/// with HTTP Basic auth (default admin/1234) and exposes two data-bearing CGI GETs: +/// +/// GET /cgi/get_sn -> {cur_sn:"..(ASCII)",sn:"..(ASCII)",cur_pass:"..(ASCII)"} +/// GET /cgi/get_gpon_info -> {line_status:"5",loid_status:0,up_fec:"Disable",.., +/// temp:"67.85",voltage:"3.30",current:"28.89", +/// tx_power:"3.01",rx_power:"-17.17"} +/// +/// These responses are JavaScript object literals, not strict JSON (unquoted keys, +/// whitespace after colons, mixed value types, a terminal "(ASCII)" suffix inside +/// strings), so they are decoded by a small provider-local lenient tokenizer rather +/// than System.Text.Json. +/// +/// get_gpon_info is the required telemetry endpoint; get_sn is best-effort identity +/// enrichment only, and its body is never logged because it contains cur_pass. +/// +public sealed class ZyxelGponSfpOntProvider : IOntProvider +{ + public string ProviderKey => "zyxel-gpon-sfp"; + public string DisplayName => "Zyxel GPON-SFP (PMG3000, HTTP)"; + + private const int TimeoutSeconds = 10; + private const string SnPath = "/cgi/get_sn"; + private const string GponInfoPath = "/cgi/get_gpon_info"; + private const string DefaultUsername = "admin"; + private const string DefaultPassword = "1234"; + + // Keys in get_gpon_info that mark the response as a real PON status payload (rather + // than an HTML/login page returned with HTTP 200). Presence of at least one gates + // whether the poll is treated as valid. + private static readonly string[] RecognizedGponKeys = + { "line_status", "rx_power", "tx_power", "temp", "voltage", "current" }; + + private readonly ILogger _logger; + + public ZyxelGponSfpOntProvider(ILogger logger) + { + _logger = logger; + } + + public async Task PollAsync(OntPollContext context, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(context.Host)) + { + _logger.LogWarning("Zyxel GPON-SFP ONT poll requested but Host is empty (config {Id})", context.Id); + return null; + } + + try + { + using var client = CreateClient(context); + var baseUrl = BuildBaseUrl(context); + + // Required: optical/link telemetry. A transport/HTTP failure here propagates + // to the outer catch and yields null. + var gponBody = await client.GetStringAsync($"{baseUrl}{GponInfoPath}", cancellationToken); + + // Best-effort: serial identity. Losing this endpoint must not discard the + // optical telemetry we already have, so its failure is swallowed. The body is + // never logged, since get_sn carries cur_pass. + string? snBody = null; + try + { + snBody = await client.GetStringAsync($"{baseUrl}{SnPath}", cancellationToken); + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + _logger.LogDebug(ex, + "Zyxel GPON-SFP ONT {Name}: serial enrichment (get_sn) failed, continuing with optical telemetry", + context.Name); + } + + var stats = new OntStats + { + Timestamp = DateTime.UtcNow, + DeviceHost = context.ConfiguredHost ?? context.Host, + DeviceName = context.Name, + }; + + if (!ApplyResponses(snBody, gponBody, stats)) + { + _logger.LogWarning( + "Zyxel GPON-SFP ONT {Name}: get_gpon_info did not contain recognized PON status fields", + context.Name); + return null; + } + + _logger.LogDebug( + "Zyxel GPON-SFP ONT {Name} polled: Rx={Rx} dBm, Tx={Tx} dBm, Link={Link}, SN={Sn}", + context.Name, stats.RxPowerDbm?.ToString("F2") ?? "-", + stats.TxPowerDbm?.ToString("F2") ?? "-", stats.LinkState ?? "-", stats.VendorSn ?? "-"); + + return stats; + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + _logger.LogWarning(ex, "Error polling Zyxel GPON-SFP ONT {Name} at {Host}", + context.Name, context.ConfiguredHost ?? context.Host); + return null; + } + } + + public async Task<(bool Success, string Message)> TestConnectionAsync( + OntPollContext context, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(context.Host)) + return (false, "Host is empty"); + + try + { + using var client = CreateClient(context); + var baseUrl = BuildBaseUrl(context); + + // Exercise the telemetry endpoint monitoring actually needs, so a success cannot + // be reported while get_gpon_info is broken. + using var response = await client.GetAsync($"{baseUrl}{GponInfoPath}", cancellationToken); + if (!response.IsSuccessStatusCode) + { + return response.StatusCode switch + { + HttpStatusCode.Unauthorized => + (false, "Authentication failed - check username/password (default is admin/1234)"), + HttpStatusCode.Forbidden => + (false, "Access denied (HTTP 403)"), + _ => (false, $"Device returned HTTP {(int)response.StatusCode} {response.ReasonPhrase}"), + }; + } + + var gponBody = await response.Content.ReadAsStringAsync(cancellationToken); + + var stats = new OntStats(); + if (!ApplyResponses(null, gponBody, stats)) + return (false, "Connected but response did not contain the expected GPON status fields"); + + return (true, + $"Connected (HTTP) - RX: {stats.RxPowerDbm?.ToString("F2") ?? "?"} dBm, Link: {stats.LinkState ?? "?"}"); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // HttpClient.Timeout elapsed - surfaces as a cancellation not tied to our token. + return (false, "Connection timed out"); + } + catch (OperationCanceledException) + { + throw; // caller-requested cancellation + } + catch (HttpRequestException ex) + { + return (false, $"Connection failed - device unreachable ({ex.Message})"); + } + catch (Exception ex) + { + return (false, $"Connection failed: {ex.Message}"); + } + } + + /// + /// Maps the two CGI responses onto . get_gpon_info is required: + /// this returns false (and leaves untouched) if it does not parse + /// or contains no recognized PON status field, so the caller can drop the poll. get_sn is + /// optional identity enrichment and its failure never affects the mapped optical telemetry. + /// Never throws for bad device data. + /// + internal static bool ApplyResponses(string? snBody, string gponBody, OntStats stats) + { + if (!TryParseObjectLiteral(gponBody, out var gpon) || !RecognizedGponKeys.Any(gpon.ContainsKey)) + return false; + + // Identity seeds, applied only once get_gpon_info is confirmed to be a real payload. + stats.VendorName = "Zyxel"; + stats.DeviceModel = "Zyxel PMG3000"; + stats.PonType = "GPON"; + + // line_status is the raw ONU state ordinal (device returns "5"); prefix "O" so it + // reuses the shared O1-O7 parser. + if (gpon.TryGetValue("line_status", out var lineStatus)) + { + var state = PonLinkStateExtensions.ParsePonLinkState("O" + lineStatus.Trim()); + stats.PonLinkStatus = state; + stats.LinkState = state.ToDisplayString(); + stats.OperationalStatus = state switch + { + PonLinkState.Operation => "Up", + PonLinkState.Unknown => null, + _ => "Down", + }; + } + + stats.RxPowerDbm = ParseDouble(GetValue(gpon, "rx_power")) ?? stats.RxPowerDbm; + stats.TxPowerDbm = ParseDouble(GetValue(gpon, "tx_power")) ?? stats.TxPowerDbm; + stats.TemperatureC = ParseDouble(GetValue(gpon, "temp")) ?? stats.TemperatureC; + stats.VoltageV = ParseDouble(GetValue(gpon, "voltage")) ?? stats.VoltageV; + stats.BiasMa = ParseDouble(GetValue(gpon, "current")) ?? stats.BiasMa; + + // up_fec/down_fec are FEC enablement flags, not cumulative error counts, so they are + // deliberately not mapped to OntStats.FecErrors. + + if (snBody is not null && TryParseObjectLiteral(snBody, out var sn)) + { + var serial = CleanSerial(GetValue(sn, "cur_sn") ?? GetValue(sn, "sn")); + if (!string.IsNullOrEmpty(serial)) + stats.VendorSn = serial; + } + + return true; + } + + /// + /// Hand-rolled linear tokenizer for the stick's JavaScript-object-literal responses. + /// Handles optional leading whitespace/BOM, identifier or quoted keys, single/double + /// quoted string values with escapes, bare scalar tokens (0, -17.17, Disable, null), + /// an optional trailing comma, a required closing brace, and rejects unexplained + /// trailing content. Duplicate keys resolve to the last value. Returns false (with an + /// empty dictionary) for any malformed input; it never throws. A regex is deliberately + /// avoided: it is fragile around quoted delimiters and escapes, and the repo forbids + /// adding a permissive-JSON dependency. + /// + internal static bool TryParseObjectLiteral(string body, out IReadOnlyDictionary fields) + { + var result = new Dictionary(StringComparer.Ordinal); + fields = result; + + if (TryParseInto(body, result)) + return true; + + result.Clear(); // no partial results leak out on malformed input + return false; + } + + private static bool TryParseInto(string body, Dictionary result) + { + if (body is null) + return false; + + var i = 0; + var n = body.Length; + + if (n > 0 && body[0] == '') + i = 1; + + SkipWhitespace(body, ref i); + if (i >= n || body[i] != '{') + return false; + i++; + + SkipWhitespace(body, ref i); + if (i < n && body[i] == '}') + { + i++; + SkipWhitespace(body, ref i); + return i == n; + } + + while (true) + { + SkipWhitespace(body, ref i); + if (!TryReadKey(body, ref i, out var key)) + return false; + + SkipWhitespace(body, ref i); + if (i >= n || body[i] != ':') + return false; + i++; + + SkipWhitespace(body, ref i); + if (!TryReadValue(body, ref i, out var value)) + return false; + + result[key] = value; // last write wins on duplicate keys + + SkipWhitespace(body, ref i); + if (i >= n) + return false; // missing closing brace + + if (body[i] == ',') + { + i++; + SkipWhitespace(body, ref i); + if (i < n && body[i] == '}') // trailing comma before close + { + i++; + SkipWhitespace(body, ref i); + return i == n; + } + continue; + } + + if (body[i] == '}') + { + i++; + SkipWhitespace(body, ref i); + return i == n; + } + + return false; // unexpected character between pairs + } + } + + private static bool TryReadKey(string body, ref int i, out string key) + { + key = ""; + if (i >= body.Length) + return false; + + var c = body[i]; + if (c is '"' or '\'') + return TryReadQuoted(body, ref i, out key); + + var start = i; + while (i < body.Length) + { + var ch = body[i]; + if (char.IsWhiteSpace(ch) || ch is ':' or ',' or '{' or '}' or '"' or '\'') + break; + i++; + } + + if (i == start) + return false; + + key = body[start..i]; + return true; + } + + private static bool TryReadValue(string body, ref int i, out string value) + { + value = ""; + if (i >= body.Length) + return false; + + var c = body[i]; + if (c is '"' or '\'') + return TryReadQuoted(body, ref i, out value); + + var start = i; + while (i < body.Length) + { + var ch = body[i]; + if (char.IsWhiteSpace(ch) || ch is ',' or '}') + break; + i++; + } + + if (i == start) + return false; // empty/missing value + + value = body[start..i]; + return true; + } + + private static bool TryReadQuoted(string body, ref int i, out string value) + { + value = ""; + var quote = body[i]; + i++; // opening quote + + var sb = new StringBuilder(); + while (i < body.Length) + { + var ch = body[i]; + if (ch == '\\') + { + i++; + if (i >= body.Length) + return false; // dangling escape + sb.Append(body[i] switch + { + 'n' => '\n', + 't' => '\t', + 'r' => '\r', + 'b' => '\b', + 'f' => '\f', + var other => other, // includes \" \' \\ \/ and any unknown escape + }); + i++; + continue; + } + + if (ch == quote) + { + i++; // closing quote + value = sb.ToString(); + return true; + } + + sb.Append(ch); + i++; + } + + return false; // unterminated quote + } + + private static void SkipWhitespace(string body, ref int i) + { + while (i < body.Length && char.IsWhiteSpace(body[i])) + i++; + } + + /// + /// Trims the value, strips a terminal "(ASCII)" suffix (only when terminal), and returns + /// null if nothing meaningful remains. Applied to serial fields only. + /// + private static string? CleanSerial(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + + var s = raw.Trim(); + const string suffix = "(ASCII)"; + if (s.EndsWith(suffix, StringComparison.Ordinal)) + s = s[..^suffix.Length].Trim(); + + return s.Length == 0 ? null : s; + } + + private static string? GetValue(IReadOnlyDictionary fields, string key) => + fields.TryGetValue(key, out var v) ? v : null; + + private static double? ParseDouble(string? text) => + double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var val) && double.IsFinite(val) + ? val + : null; + + /// + /// Builds an HttpClient with preemptive HTTP Basic auth from the context credentials + /// (defaulting to admin/1234 when blank) and a 10-second per-request timeout. No internal + /// retry: the polling schedule is the retry mechanism and the stick is resource-constrained. + /// + internal static HttpClient CreateClient(OntPollContext context) + { + var user = string.IsNullOrWhiteSpace(context.Username) ? DefaultUsername : context.Username; + var pass = string.IsNullOrWhiteSpace(context.Password) ? DefaultPassword : context.Password; + var token = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{pass}")); + + var client = new HttpClient { Timeout = TimeSpan.FromSeconds(TimeoutSeconds) }; + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", token); + return client; + } + + private static string BuildBaseUrl(OntPollContext context) + { + var port = context.Port > 0 ? context.Port : 80; + var portSuffix = port == 80 ? "" : $":{port}"; + return $"http://{context.Host}{portSuffix}"; + } +} diff --git a/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs b/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs new file mode 100644 index 000000000..2c1c0f446 --- /dev/null +++ b/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs @@ -0,0 +1,398 @@ +using System.Text; +using FluentAssertions; +using NetworkOptimizer.Monitoring.Models; +using NetworkOptimizer.Monitoring.Providers; +using NetworkOptimizer.Web.Services.OntProviders; +using Xunit; + +namespace NetworkOptimizer.Web.Tests; + +public class ZyxelGponSfpOntProviderTests +{ + // Captured verbatim from the live Zyxel PMG3000 GPON-SFP stick on 2026-07-20, with the + // real cloned serial and cur_pass scrubbed to placeholders. These are JavaScript object + // literals, not strict JSON: unquoted keys, whitespace after colons, mixed value types, + // and a terminal "(ASCII)" suffix inside string values. + private const string SnBody = + """{cur_sn:"TW00ABCD1234(ASCII)",sn: "TW00ABCD1234(ASCII)",cur_pass:"placeholder(ASCII)"}"""; + + private const string GponInfoBody = + """{line_status:"5",loid_status:0,up_fec:"Disable",down_fec:"Disable",encrypt:"Disable",temp:"67.85",voltage:"3.30",current:"28.89",tx_power:"3.01",rx_power:"-17.17"}"""; + + // --- Mapping cases ------------------------------------------------------- + + [Fact] + public void ApplyResponses_HappyPath_MapsAllFieldsWithUnits() + { + var stats = new OntStats(); + + var ok = ZyxelGponSfpOntProvider.ApplyResponses(SnBody, GponInfoBody, stats); + + ok.Should().BeTrue(); + stats.RxPowerDbm.Should().BeApproximately(-17.17, 0.0001); + stats.TxPowerDbm.Should().BeApproximately(3.01, 0.0001); + stats.TemperatureC.Should().BeApproximately(67.85, 0.0001); + stats.VoltageV.Should().BeApproximately(3.30, 0.0001); + stats.BiasMa.Should().BeApproximately(28.89, 0.0001); + stats.PonLinkStatus.Should().Be(PonLinkState.Operation); + stats.OperationalStatus.Should().Be("Up"); + stats.LinkState.Should().Be("Connected (O5)"); + stats.VendorSn.Should().Be("TW00ABCD1234"); + stats.VendorName.Should().Be("Zyxel"); + stats.DeviceModel.Should().Be("Zyxel PMG3000"); + stats.PonType.Should().Be("GPON"); + } + + [Fact] + public void ApplyResponses_NoUpFecMappedToFecErrors() + { + var stats = new OntStats(); + + ZyxelGponSfpOntProvider.ApplyResponses(SnBody, GponInfoBody, stats); + + // up_fec/down_fec are enablement flags ("Disable"), not error counts. + stats.FecErrors.Should().BeNull(); + } + + [Fact] + public void ApplyResponses_MissingGetSn_StillMapsOptics() + { + var stats = new OntStats(); + + var ok = ZyxelGponSfpOntProvider.ApplyResponses(null, GponInfoBody, stats); + + ok.Should().BeTrue(); + stats.RxPowerDbm.Should().BeApproximately(-17.17, 0.0001); + stats.TxPowerDbm.Should().BeApproximately(3.01, 0.0001); + stats.PonLinkStatus.Should().Be(PonLinkState.Operation); + stats.VendorSn.Should().BeNull(); + } + + [Fact] + public void ApplyResponses_MalformedGetSn_DoesNotDiscardOptics() + { + var stats = new OntStats(); + + var ok = ZyxelGponSfpOntProvider.ApplyResponses("login", GponInfoBody, stats); + + ok.Should().BeTrue(); + stats.RxPowerDbm.Should().BeApproximately(-17.17, 0.0001); + stats.VendorSn.Should().BeNull(); + } + + [Fact] + public void ApplyResponses_GponInfoWithoutRecognizedField_ReturnsFalse() + { + var stats = new OntStats(); + + // Parses fine, but contains no recognized PON status field. + var ok = ZyxelGponSfpOntProvider.ApplyResponses(SnBody, """{loid_status:0,encrypt:"Disable"}""", stats); + + ok.Should().BeFalse(); + stats.RxPowerDbm.Should().BeNull(); + stats.VendorName.Should().BeNull(); + stats.DeviceModel.Should().Be(""); + } + + [Theory] + [InlineData("Login")] + [InlineData("")] + [InlineData(" ")] + [InlineData("not an object")] + public void ApplyResponses_MalformedGponInfo_ReturnsFalseWithoutThrowing(string gponBody) + { + var stats = new OntStats(); + + var act = () => ZyxelGponSfpOntProvider.ApplyResponses(SnBody, gponBody, stats); + + act.Should().NotThrow(); + ZyxelGponSfpOntProvider.ApplyResponses(SnBody, gponBody, stats).Should().BeFalse(); + stats.RxPowerDbm.Should().BeNull(); + } + + [Theory] + [InlineData("1", PonLinkState.Initial, "Down", "Initializing (O1)")] + [InlineData("2", PonLinkState.Standby, "Down", "Standby (O2)")] + [InlineData("3", PonLinkState.SerialNumber, "Down", "Authenticating (O3)")] + [InlineData("4", PonLinkState.Ranging, "Down", "Ranging (O4)")] + [InlineData("5", PonLinkState.Operation, "Up", "Connected (O5)")] + [InlineData("6", PonLinkState.Popup, "Down", "Signal Lost (O6)")] + [InlineData("7", PonLinkState.EmergencyStop, "Down", "Disabled (O7)")] + public void ApplyResponses_LineStatus_MapsToPonState( + string lineStatus, PonLinkState expectedState, string expectedOp, string expectedLabel) + { + var stats = new OntStats(); + var gpon = $$"""{line_status:"{{lineStatus}}",rx_power:"-17.17"}"""; + + var ok = ZyxelGponSfpOntProvider.ApplyResponses(null, gpon, stats); + + ok.Should().BeTrue(); + stats.PonLinkStatus.Should().Be(expectedState); + stats.OperationalStatus.Should().Be(expectedOp); + stats.LinkState.Should().Be(expectedLabel); + } + + [Theory] + [InlineData("0")] + [InlineData("9")] + public void ApplyResponses_LineStatusUnknown_LeavesOperationalStatusNull(string lineStatus) + { + var stats = new OntStats(); + var gpon = $$"""{line_status:"{{lineStatus}}",rx_power:"-17.17"}"""; + + var ok = ZyxelGponSfpOntProvider.ApplyResponses(null, gpon, stats); + + ok.Should().BeTrue(); + stats.PonLinkStatus.Should().Be(PonLinkState.Unknown); + stats.OperationalStatus.Should().BeNull(); + stats.LinkState.Should().Be("Unknown"); + } + + [Fact] + public void ApplyResponses_QuotedAndBareLineStatus_BothMapEqually() + { + var quoted = new OntStats(); + var bare = new OntStats(); + + ZyxelGponSfpOntProvider.ApplyResponses(null, """{line_status:"5",rx_power:"-17.17"}""", quoted); + ZyxelGponSfpOntProvider.ApplyResponses(null, """{line_status:5,rx_power:-17.17}""", bare); + + quoted.PonLinkStatus.Should().Be(PonLinkState.Operation); + bare.PonLinkStatus.Should().Be(PonLinkState.Operation); + bare.RxPowerDbm.Should().BeApproximately(-17.17, 0.0001); + } + + // --- Serial handling ----------------------------------------------------- + + [Fact] + public void ApplyResponses_PrefersCurSnOverSn() + { + var stats = new OntStats(); + var sn = """{cur_sn:"AAAA1111(ASCII)",sn:"BBBB2222(ASCII)"}"""; + + ZyxelGponSfpOntProvider.ApplyResponses(sn, GponInfoBody, stats); + + stats.VendorSn.Should().Be("AAAA1111"); + } + + [Fact] + public void ApplyResponses_FallsBackToSnWhenCurSnMissing() + { + var stats = new OntStats(); + var sn = """{sn:"BBBB2222(ASCII)",cur_pass:"placeholder(ASCII)"}"""; + + ZyxelGponSfpOntProvider.ApplyResponses(sn, GponInfoBody, stats); + + stats.VendorSn.Should().Be("BBBB2222"); + } + + [Fact] + public void ApplyResponses_StripsAsciiSuffixOnlyWhenTerminal() + { + var stats = new OntStats(); + // "(ASCII)" appears mid-value, so it must be preserved. + var sn = """{cur_sn:"AB(ASCII)CD"}"""; + + ZyxelGponSfpOntProvider.ApplyResponses(sn, GponInfoBody, stats); + + stats.VendorSn.Should().Be("AB(ASCII)CD"); + } + + // --- Parser cases -------------------------------------------------------- + + [Fact] + public void TryParseObjectLiteral_SnKeyNotMatchedInsideCurSn() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral(SnBody, out var fields).Should().BeTrue(); + + fields.Should().ContainKey("cur_sn"); + fields.Should().ContainKey("sn"); + fields["cur_sn"].Should().Be("TW00ABCD1234(ASCII)"); + fields["sn"].Should().Be("TW00ABCD1234(ASCII)"); + } + + [Fact] + public void TryParseObjectLiteral_KeyLikeStringInsideValueIsNotAKey() + { + // A value that itself looks like "key:value" must stay a single value. + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{note:"sn:should_not_be_key",rx_power:"-17.17"}""", out var fields) + .Should().BeTrue(); + + fields["note"].Should().Be("sn:should_not_be_key"); + fields.Should().NotContainKey("should_not_be_key"); + fields["rx_power"].Should().Be("-17.17"); + } + + [Fact] + public void TryParseObjectLiteral_WhitespaceAfterColon() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{sn: "value"}""", out var fields).Should().BeTrue(); + + fields["sn"].Should().Be("value"); + } + + [Fact] + public void TryParseObjectLiteral_LeadingWhitespaceAndBom() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral(" \n {a:1}", out var fields).Should().BeTrue(); + + fields["a"].Should().Be("1"); + } + + [Fact] + public void TryParseObjectLiteral_QuotedKeys() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{"line_status":"5","rx_power":"-17.17"}""", out var fields) + .Should().BeTrue(); + + fields["line_status"].Should().Be("5"); + fields["rx_power"].Should().Be("-17.17"); + } + + [Fact] + public void TryParseObjectLiteral_SingleQuotedValues() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{a:'hello',b:'wo,rld'}""", out var fields).Should().BeTrue(); + + fields["a"].Should().Be("hello"); + fields["b"].Should().Be("wo,rld"); + } + + [Theory] + [InlineData("""{v:-17.17}""", "-17.17")] + [InlineData("""{v:3.30}""", "3.30")] + [InlineData("""{v:1e3}""", "1e3")] + [InlineData("""{v:0}""", "0")] + public void TryParseObjectLiteral_BareNumbers(string body, string expected) + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral(body, out var fields).Should().BeTrue(); + + fields["v"].Should().Be(expected); + } + + [Theory] + [InlineData("NaN")] + [InlineData("Infinity")] + [InlineData("-Infinity")] + public void ApplyResponses_NonFiniteNumbers_RejectedAsNull(string value) + { + var stats = new OntStats(); + var gpon = $$"""{line_status:"5",rx_power:{{value}}}"""; + + var ok = ZyxelGponSfpOntProvider.ApplyResponses(null, gpon, stats); + + ok.Should().BeTrue(); // line_status keeps it a recognized response + stats.RxPowerDbm.Should().BeNull(); + } + + [Fact] + public void ApplyResponses_ExponentAndNegativeNumbersParse() + { + var stats = new OntStats(); + + ZyxelGponSfpOntProvider.ApplyResponses(null, """{line_status:"5",rx_power:-1.5e1,tx_power:2}""", stats); + + stats.RxPowerDbm.Should().BeApproximately(-15.0, 0.0001); + stats.TxPowerDbm.Should().BeApproximately(2.0, 0.0001); + } + + [Fact] + public void TryParseObjectLiteral_ValuesWithCommasColonsAndEscapes() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral( + """{a:"x,y:z",b:"quote\"here",c:"back\\slash"}""", out var fields).Should().BeTrue(); + + fields["a"].Should().Be("x,y:z"); + fields["b"].Should().Be("quote\"here"); + fields["c"].Should().Be("back\\slash"); + } + + [Fact] + public void TryParseObjectLiteral_TrailingComma() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{a:1,b:2,}""", out var fields).Should().BeTrue(); + + fields["a"].Should().Be("1"); + fields["b"].Should().Be("2"); + } + + [Fact] + public void TryParseObjectLiteral_EmptyObject() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("{}", out var fields).Should().BeTrue(); + + fields.Should().BeEmpty(); + } + + [Fact] + public void TryParseObjectLiteral_UnknownFieldsAndNullAndBool() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{x:null,y:true,z:false}""", out var fields).Should().BeTrue(); + + fields["x"].Should().Be("null"); + fields["y"].Should().Be("true"); + fields["z"].Should().Be("false"); + } + + [Fact] + public void TryParseObjectLiteral_DuplicateKeysLastWins() + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral("""{a:1,a:2,a:3}""", out var fields).Should().BeTrue(); + + fields["a"].Should().Be("3"); + } + + [Theory] + [InlineData("""{a 1}""")] // missing colon + [InlineData("""{a:"unterminated}""")] // unterminated quote + [InlineData("""{a:1""")] // missing closing brace + [InlineData("""{a:1}extra""")] // trailing content + [InlineData("not an object")] // no opening brace + [InlineData("")] // empty + public void TryParseObjectLiteral_MalformedInput_ReturnsFalse(string body) + { + ZyxelGponSfpOntProvider.TryParseObjectLiteral(body, out var fields).Should().BeFalse(); + fields.Should().BeEmpty(); + } + + // --- Client / auth ------------------------------------------------------- + + [Fact] + public void CreateClient_SetsPreemptiveBasicAuthFromContext() + { + var context = new OntPollContext + { + Id = 1, + Name = "Zyxel", + Host = "10.10.1.1", + Username = "user", + Password = "secret", + }; + + using var client = ZyxelGponSfpOntProvider.CreateClient(context); + + client.DefaultRequestHeaders.Authorization.Should().NotBeNull(); + client.DefaultRequestHeaders.Authorization!.Scheme.Should().Be("Basic"); + Encoding.UTF8.GetString(Convert.FromBase64String(client.DefaultRequestHeaders.Authorization.Parameter!)) + .Should().Be("user:secret"); + client.Timeout.Should().Be(TimeSpan.FromSeconds(10)); + } + + [Fact] + public void CreateClient_DefaultsToAdmin1234WhenBlank() + { + var context = new OntPollContext + { + Id = 1, + Name = "Zyxel", + Host = "10.10.1.1", + Username = "", + Password = "", + }; + + using var client = ZyxelGponSfpOntProvider.CreateClient(context); + + Encoding.UTF8.GetString(Convert.FromBase64String(client.DefaultRequestHeaders.Authorization!.Parameter!)) + .Should().Be("admin:1234"); + } +} From 82573b399bdfb08c7c9dcebfeb2b1112d406d729 Mon Sep 17 00:00:00 2001 From: Optic00 Date: Mon, 20 Jul 2026 19:17:07 +0200 Subject: [PATCH 13/23] Address Codex review: robust get_sn timeout + strict line_status - get_sn (best-effort serial) timeout no longer discards optical telemetry: an HttpClient.Timeout surfaces as OperationCanceledException with the caller token not cancelled; rethrow only on genuine caller cancellation, otherwise swallow (inner) or return null (outer) per the IOntProvider contract. - line_status is matched as an exact single ordinal 1-7 (new MapLineStatus) instead of the shared substring parser, so a malformed value like "51" no longer substring-matches O5 and gets misreported as a healthy Up link. - Regression tests for both. --- .../OntProviders/ZyxelGponSfpOntProvider.cs | 38 ++++++++++++++++--- .../ZyxelGponSfpOntProviderTests.cs | 16 ++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs b/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs index 01aadec29..1eab395bb 100644 --- a/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs +++ b/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs @@ -75,9 +75,15 @@ public ZyxelGponSfpOntProvider(ILogger logger) { snBody = await client.GetStringAsync($"{baseUrl}{SnPath}", cancellationToken); } - catch (OperationCanceledException) { throw; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; // genuine caller-requested cancellation + } catch (Exception ex) { + // Includes an HttpClient.Timeout (surfaces as OperationCanceledException with our + // token NOT cancelled): get_sn is best-effort, so a slow/hung serial endpoint must + // not discard the optical telemetry we already have. _logger.LogDebug(ex, "Zyxel GPON-SFP ONT {Name}: serial enrichment (get_sn) failed, continuing with optical telemetry", context.Name); @@ -105,9 +111,14 @@ public ZyxelGponSfpOntProvider(ILogger logger) return stats; } - catch (OperationCanceledException) { throw; } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; // genuine caller-requested cancellation + } catch (Exception ex) { + // A transport failure or an HttpClient.Timeout (OperationCanceledException with our + // token not cancelled) yields null per the IOntProvider contract, not an exception. _logger.LogWarning(ex, "Error polling Zyxel GPON-SFP ONT {Name} at {Host}", context.Name, context.ConfiguredHost ?? context.Host); return null; @@ -185,11 +196,10 @@ internal static bool ApplyResponses(string? snBody, string gponBody, OntStats st stats.DeviceModel = "Zyxel PMG3000"; stats.PonType = "GPON"; - // line_status is the raw ONU state ordinal (device returns "5"); prefix "O" so it - // reuses the shared O1-O7 parser. + // line_status is the raw ONU state ordinal (device returns "5"). if (gpon.TryGetValue("line_status", out var lineStatus)) { - var state = PonLinkStateExtensions.ParsePonLinkState("O" + lineStatus.Trim()); + var state = MapLineStatus(lineStatus); stats.PonLinkStatus = state; stats.LinkState = state.ToDisplayString(); stats.OperationalStatus = state switch @@ -410,6 +420,24 @@ private static void SkipWhitespace(string body, ref int i) i++; } + /// + /// Maps the device's line_status to an ITU ONU state. The stick reports a single ordinal + /// "1".."7"; matched exactly (not via substring), so an unexpected/malformed value such as + /// "51" or "15" resolves to Unknown rather than being misread as a healthy O5 - reporting a + /// bad status as Up could suppress a genuine down alert. + /// + internal static PonLinkState MapLineStatus(string? raw) => raw?.Trim() switch + { + "1" => PonLinkState.Initial, + "2" => PonLinkState.Standby, + "3" => PonLinkState.SerialNumber, + "4" => PonLinkState.Ranging, + "5" => PonLinkState.Operation, + "6" => PonLinkState.Popup, + "7" => PonLinkState.EmergencyStop, + _ => PonLinkState.Unknown, + }; + /// /// Trims the value, strips a terminal "(ASCII)" suffix (only when terminal), and returns /// null if nothing meaningful remains. Applied to serial fields only. diff --git a/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs b/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs index 2c1c0f446..5766754a2 100644 --- a/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/ZyxelGponSfpOntProviderTests.cs @@ -135,6 +135,11 @@ public void ApplyResponses_LineStatus_MapsToPonState( [Theory] [InlineData("0")] [InlineData("9")] + [InlineData("51")] // must NOT substring-match O5 -> Operation + [InlineData("15")] // must NOT substring-match O1 -> Initial + [InlineData("55")] + [InlineData("O5")] // literal, not a bare ordinal + [InlineData("")] public void ApplyResponses_LineStatusUnknown_LeavesOperationalStatusNull(string lineStatus) { var stats = new OntStats(); @@ -148,6 +153,17 @@ public void ApplyResponses_LineStatusUnknown_LeavesOperationalStatusNull(string stats.LinkState.Should().Be("Unknown"); } + [Theory] + [InlineData("51")] + [InlineData("15")] + [InlineData("57")] + public void MapLineStatus_MultiDigit_IsNotMisreadAsHealthy(string lineStatus) + { + // Regression: substring parsing would have turned "51" into O5/Operation and reported + // the link Up, potentially suppressing a down alert. + ZyxelGponSfpOntProvider.MapLineStatus(lineStatus).Should().Be(PonLinkState.Unknown); + } + [Fact] public void ApplyResponses_QuotedAndBareLineStatus_BothMapEqually() { From 7e5aec543120e1534f0ced82bed9aae43ee9945b Mon Sep 17 00:00:00 2001 From: Optic00 Date: Mon, 20 Jul 2026 19:44:36 +0200 Subject: [PATCH 14/23] Settings: suggest provider default host for ONT (Zyxel -> 10.10.1.1) The Add/Edit ONT form showed the same generic 192.168.1.254 host placeholder for every provider. Make it provider-aware, mirroring the existing port default: selecting the Zyxel GPON-SFP provider now placeholders and prefills (when the host field is empty) the stick's fixed management IP 10.10.1.1, so the correct value is suggested instead of a meaningless generic one. Other providers keep the generic placeholder and are never prefilled. --- .../Components/Pages/Settings.razor | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index 073284b9d..4dfaa6054 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -1574,7 +1574,7 @@
- + Behind your WAN and unreachable for polling? Set up a monitoring interface → @@ -5844,9 +5844,20 @@ private void OnOntProviderChanged(ChangeEventArgs e) { + var previousProvider = _editingOnt.Provider; var provider = e.Value?.ToString() ?? "att-gateway"; _editingOnt.Provider = provider; _editingOnt.Port = GetOntProviderDefaultPort(provider); + + // Suggest the provider's known management host, mirroring the port default. Fill when the + // field is empty, and replace a host that was auto-filled from the previous provider's + // default (so a stale suggestion is not saved for a different device) - but never + // overwrite a value the user actually typed. + if (string.IsNullOrWhiteSpace(_editingOnt.Host) || + _editingOnt.Host == GetOntProviderDefaultHost(previousProvider)) + { + _editingOnt.Host = GetOntProviderDefaultHost(provider); + } } private static int GetOntProviderDefaultPort(string provider) => provider switch @@ -5856,6 +5867,21 @@ _ => 80, }; + // Known default management address for providers whose stick/box ships on a fixed IP; + // blank when there is no single sensible default (so nothing is prefilled). + private static string GetOntProviderDefaultHost(string provider) => provider switch + { + "zyxel-gpon-sfp" => "10.10.1.1", + _ => "", + }; + + // Host-field placeholder: the provider's known default when there is one, else a generic example. + private static string GetOntHostPlaceholder(string provider) + { + var suggested = GetOntProviderDefaultHost(provider); + return string.IsNullOrEmpty(suggested) ? "192.168.1.254" : suggested; + } + private static string GetOntProviderDisplayName(string provider) => provider switch { "att-gateway" => "AT&T Gateway (HTTP)", From 4d6eb9ec5fa969367fb1b9c7e45967e0f8c82ea8 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 13:32:57 -0500 Subject: [PATCH 15/23] ONT: per-provider default management host + Zyxel option rename Suggest a known default Host/IP when selecting an ONT provider whose device ships on a fixed management address (AT&T 192.168.1.254, 8311 192.168.11.1, Quantum 192.168.0.1, Telekom/Nokia 192.168.100.1), matching the existing port-default behavior. Realtek, netopt-custom and generic stay blank (no single sensible default). Rename the Zyxel option/display name to "Zyxel GPON-SFP PMG3000 (HTTP)" so only the transport sits in the trailing parenthetical. --- .../Components/Pages/Settings.razor | 12 ++++++++++-- .../Services/OntProviders/ZyxelGponSfpOntProvider.cs | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index 547f820a4..ba88f1765 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -1565,7 +1565,7 @@ - + @@ -5913,7 +5913,15 @@ // blank when there is no single sensible default (so nothing is prefilled). private static string GetOntProviderDefaultHost(string provider) => provider switch { + "att-gateway" => "192.168.1.254", + "8311" => "192.168.11.1", + "quantum-q1000k" => "192.168.0.1", + "telekom-modem2" => "192.168.100.1", + "nokia-xs010x-q" => "192.168.100.1", "zyxel-gpon-sfp" => "10.10.1.1", + // realtek-ont, netopt-custom and generic-http-ont have no single sensible + // default (vendor-dependent stick, gateway-served JSON, or generic), so nothing + // is prefilled and the generic example placeholder is shown. _ => "", }; @@ -5932,7 +5940,7 @@ "quantum-q1000k" => "Quantum Fiber Q1000K (HTTP)", "telekom-modem2" => "Telekom Glasfaser-Modem 2 (HTTP)", "nokia-xs010x-q" => "Nokia XS-010X-Q (HTTP)", - "zyxel-gpon-sfp" => "Zyxel GPON-SFP (PMG3000, HTTP)", + "zyxel-gpon-sfp" => "Zyxel GPON-SFP PMG3000 (HTTP)", "netopt-custom" => "Network Optimizer Custom (HTTP JSON)", "generic-http-ont" => "Generic HTTP ONT", _ => provider, diff --git a/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs b/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs index 1eab395bb..f8b3ce265 100644 --- a/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs +++ b/src/NetworkOptimizer.Web/Services/OntProviders/ZyxelGponSfpOntProvider.cs @@ -29,7 +29,7 @@ namespace NetworkOptimizer.Web.Services.OntProviders; public sealed class ZyxelGponSfpOntProvider : IOntProvider { public string ProviderKey => "zyxel-gpon-sfp"; - public string DisplayName => "Zyxel GPON-SFP (PMG3000, HTTP)"; + public string DisplayName => "Zyxel GPON-SFP PMG3000 (HTTP)"; private const int TimeoutSeconds = 10; private const string SnPath = "/cgi/get_sn"; From af3afc0f928153ecbc597491277bc8524a89e9a0 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 13:37:36 -0500 Subject: [PATCH 16/23] ONT form + Add-a-Site wizard UX fixes - Prefill the ONT Host for the default provider. att-gateway is selected by default and never fires OnOntProviderChanged, so its known host was only shown as a placeholder, not filled. Centralize the fresh-form config in NewDefaultOnt() and seed Host/Port from the provider defaults. - Generic host placeholder now 192.168.100.1 (the most common ONT/modem management address) instead of 192.168.1.254. - Add-a-Site wizard: reset the completed "... is ready" screen back to a fresh form when a site's Configuration is opened from the Sites table, so the stale success state does not linger. No-op mid-setup, so input is never discarded. --- .../Components/Pages/Settings.razor | 21 +++++++++++++++--- .../Components/Shared/SiteSetupWizard.razor | 22 ++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index ba88f1765..cab228933 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -3734,6 +3734,9 @@ _newAgentToken = null; if (_expandedSiteId != null) { + // Opening a site's Configuration clears the Add-a-Site wizard's completed + // success screen so it does not linger next to the site being configured. + _setupWizard?.ResetIfComplete(); await LoadAgentsAsync(); var site = _sites.FirstOrDefault(s => s.Id == siteId); if (site != null) @@ -3981,7 +3984,7 @@ // External ONT Monitoring private List _ontConfigs = new(); private List _attachableSfps = new(); - private OntConfiguration _editingOnt = new() { Provider = "att-gateway", Port = 80, PollingIntervalSeconds = 300, Enabled = true }; + private OntConfiguration _editingOnt = NewDefaultOnt(); private bool _ontFormVisible; private bool _savingOnt; private string? _ontTestResult; @@ -5874,10 +5877,22 @@ private void CancelEditOnt() { - _editingOnt = new OntConfiguration { Provider = "att-gateway", Port = 80, PollingIntervalSeconds = 300, Enabled = true }; + _editingOnt = NewDefaultOnt(); _ontFormVisible = false; } + // A fresh Add-ONT form. The default provider (att-gateway) never fires + // OnOntProviderChanged, so its known management host is prefilled here too - + // otherwise the default selection would show only the placeholder, not a value. + private static OntConfiguration NewDefaultOnt() => new() + { + Provider = "att-gateway", + Host = GetOntProviderDefaultHost("att-gateway"), + Port = GetOntProviderDefaultPort("att-gateway"), + PollingIntervalSeconds = 300, + Enabled = true, + }; + private void OnOntProviderChanged(ChangeEventArgs e) { var previousProvider = _editingOnt.Provider; @@ -5929,7 +5944,7 @@ private static string GetOntHostPlaceholder(string provider) { var suggested = GetOntProviderDefaultHost(provider); - return string.IsNullOrEmpty(suggested) ? "192.168.1.254" : suggested; + return string.IsNullOrEmpty(suggested) ? "192.168.100.1" : suggested; } private static string GetOntProviderDisplayName(string provider) => provider switch diff --git a/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor b/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor index 6185df9db..35fb0a4cc 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor @@ -446,6 +446,27 @@ private async Task ResetAsync() { await RefreshLimitAsync(); + ResetForm(); + await OnSitesChanged.InvokeAsync(); + } + + /// + /// Clears the completed ("... is ready") success screen back to a fresh Add-a-Site + /// form. Called when the user opens a site's Configuration from the Sites table, so + /// the stale success state does not linger. No-op while a site is mid-setup, so + /// in-progress input is never discarded. + /// + public void ResetIfComplete() + { + if (_step != 4) + return; + + ResetForm(); + StateHasChanged(); + } + + private void ResetForm() + { _step = 1; _site = null; _name = ""; @@ -460,6 +481,5 @@ _featureLanSpeedTest = false; _featureProxy = false; _message = ""; - await OnSitesChanged.InvokeAsync(); } } From 4b8485d121be16baa29416d45af07ab7fcc2d263 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 14:48:54 -0500 Subject: [PATCH 17/23] Multi-Site: dismiss X on the expanded site configuration row Add a close button to the upper-right of the site config panel (Settings - Multi-Site, the row that opens from Configuration) so it can be dismissed without hunting back for the Configuration toggle. Collapses via the existing ToggleAgents handler. --- .../Components/Pages/Settings.razor | 1 + src/NetworkOptimizer.Web/wwwroot/css/app.css | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index cab228933..da4d445f9 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -3168,6 +3168,7 @@
+
Agent
@if (agents.Count > 0) diff --git a/src/NetworkOptimizer.Web/wwwroot/css/app.css b/src/NetworkOptimizer.Web/wwwroot/css/app.css index 5ca1cca61..5f31e63c8 100644 --- a/src/NetworkOptimizer.Web/wwwroot/css/app.css +++ b/src/NetworkOptimizer.Web/wwwroot/css/app.css @@ -17906,6 +17906,7 @@ body.kiosk-mode .status-indicators { } .site-config-panel { + position: relative; display: flex; flex-direction: column; gap: 0.85rem; @@ -17915,6 +17916,30 @@ body.kiosk-mode .status-indicators { border-radius: var(--border-radius-lg); } +.site-config-dismiss { + position: absolute; + top: 0.5rem; + right: 0.5rem; + display: flex; + align-items: center; + justify-content: center; + width: 1.75rem; + height: 1.75rem; + padding: 0; + border: none; + border-radius: var(--border-radius); + background: transparent; + color: var(--text-muted); + font-size: 1.25rem; + line-height: 1; + cursor: pointer; +} + +.site-config-dismiss:hover { + background: var(--bg-hover); + color: var(--text-primary); +} + .site-config-section { padding: 0.85rem 1rem; background: var(--bg-secondary); From 2eff264bd2accec0b39238fc1f89caf247fa44e8 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 15:00:41 -0500 Subject: [PATCH 18/23] Multi-Site: top-pad the site config panel so the dismiss X has room --- src/NetworkOptimizer.Web/wwwroot/css/app.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetworkOptimizer.Web/wwwroot/css/app.css b/src/NetworkOptimizer.Web/wwwroot/css/app.css index 5f31e63c8..190985171 100644 --- a/src/NetworkOptimizer.Web/wwwroot/css/app.css +++ b/src/NetworkOptimizer.Web/wwwroot/css/app.css @@ -17911,7 +17911,7 @@ body.kiosk-mode .status-indicators { flex-direction: column; gap: 0.85rem; margin: 0.25rem 0 0.75rem; - padding: 1rem; + padding: 2.5rem 1rem 1rem; background: var(--bg-tertiary); border-radius: var(--border-radius-lg); } From 77d10665f99fe73fbcb8443114effb35d8bafeb4 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 15:05:26 -0500 Subject: [PATCH 19/23] Multi-Site: drop the Dismiss tooltip on the site config X --- src/NetworkOptimizer.Web/Components/Pages/Settings.razor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index da4d445f9..a00d9ab38 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -3168,7 +3168,7 @@
- +
Agent
@if (agents.Count > 0) From 4ffa6cd7d7b23470fb3970c659736b15f8323f87 Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:16:59 -0500 Subject: [PATCH 20/23] ISP Health: P90 jitter scoring and a caveat for off-path transit scores (#1034) * ISP Health: score jitter at P90 instead of P95 Make the jitter scoring percentile a config option (AsnJitterScoringPercentile, default 0.90) instead of a hardcoded 0.95 in ScoringJitterOf. Drives per-ASN/hop jitter scoring, the path jitter floor, absolve/witness comparisons, the ISP/transit cap, and the cards - one lever. IX Peering members are graded through the same GradeAsn path, so they move with it. P95 double-counted the jitter tail with the separate congestion detector (which already catches intermittent bursts off median jitter + p90 RTT); P90 reads the typical tail and relieves the harshness without going blind like P50. Scoring is floor-relative, so numerator and floor track the percentile together. Rename P95JitterMs -> ScoredJitterMs (percentile-neutral) and the "P95 Jitter" UI label -> "P90 Jitter". Add a CI-safe replay harness (JitterPercentileReplayTests, no-ops without ISP_HEALTH_REPLAY_DIR) that isolates the jitter arm at P50/P90/P95. * ISP Health: caveat tooltip on reach-0 transit scores below 80 A transit ASN that no monitored internet target routes through on the forward path is held at the 0.25 involvement floor. When such a network scores below 80, the score text (on both Transit Health and the Networks on Your Path card) now carries a tooltip noting the low score may just be its routers deprioritizing ICMP rather than real path trouble - it's kept in view because it may carry the return path or a failover route. Score values also get a default cursor instead of the text I-beam. * ISP Health: tighten reach-0 transit caveat tooltip copy * ISP Health: fix caveat tooltip clause join * UniFi Console Connection: gate awaiting-agent state on a configured console --- .../Shared/Monitoring/IspHealthPanel.razor | 21 ++-- .../Monitoring/IspHealth/IspHealthModels.cs | 21 +++- .../Monitoring/IspHealth/IspHealthOptions.cs | 9 ++ .../Monitoring/IspHealth/IspHealthScorer.cs | 54 +++++----- .../Services/UniFiConnectionService.cs | 7 +- src/NetworkOptimizer.Web/wwwroot/css/app.css | 9 ++ .../IspHealth/IspHealthScorerTests.cs | 4 +- .../IspHealth/JitterPercentileReplayTests.cs | 99 +++++++++++++++++++ 8 files changed, 183 insertions(+), 41 deletions(-) create mode 100644 tests/NetworkOptimizer.Web.Tests/IspHealth/JitterPercentileReplayTests.cs diff --git a/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor b/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor index 97d04355b..0b11487e8 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/Monitoring/IspHealthPanel.razor @@ -363,9 +363,9 @@ else
RTT@FormatRtt(target.RttMs) - P95 Jitter + P90 Jitter - @FormatJitter(target.P95JitterMs) + @FormatJitter(target.ScoredJitterMs) @if (target.JitterAssimilated) { i @@ -450,7 +450,7 @@ else
- @(factor.Score?.ToString() ?? "--") + @(factor.Score?.ToString() ?? "--")
@if (!string.IsNullOrEmpty(factor.Description) && !(factor.Name == "Physical Link" && r.PhysicalLinkCandidates.Count > 1)) @@ -503,7 +503,7 @@ else
@(isIspAsn ? "ISP network" : asn.AsnNumber < 0 ? "Direct peering" : "Transit") @(asn.AsnNumber > 0 ? $"· AS{asn.AsnNumber}" : "")
- @(asn.OverallScore?.ToString() ?? "--") + @(asn.OverallScore?.ToString() ?? "--")
@if (showRange) @@ -515,9 +515,9 @@ else
Mean RTT@FormatRtt(asn.MeanRttMs)
}
- P95 Jitter + P90 Jitter - @FormatJitter(asn.P95JitterMs) + @FormatJitter(asn.ScoredJitterMs) @if (asn.JitterAssimilated) { i @@ -1400,11 +1400,11 @@ else private static string FormatJitter(double? ms) => ms.HasValue ? $"{ms.Value:0.00} ms" : "--"; private static string JitterAssimilationTooltip(IspAsnHealth asn, bool isIsp) => isIsp - ? $"Showing {FormatJitter(asn.P95JitterMs)} from the cleanest network beyond your ISP, not your ISP routers' {FormatJitter(asn.RawJitterMs)}. All traffic crosses the ISP to reach it, so the ISP is no jitterier - its higher reading is likely ICMP deprioritization." - : $"Showing {FormatJitter(asn.P95JitterMs)} from a router deeper in this network, not the {FormatJitter(asn.RawJitterMs)} a closer router reported. Traffic passes through the closer router to reach it, so the lower value is the true path jitter - the closer router just deprioritizes ICMP."; + ? $"Showing {FormatJitter(asn.ScoredJitterMs)} from the cleanest network beyond your ISP, not your ISP routers' {FormatJitter(asn.RawJitterMs)}. All traffic crosses the ISP to reach it, so the ISP is no jitterier - its higher reading is likely ICMP deprioritization." + : $"Showing {FormatJitter(asn.ScoredJitterMs)} from a router deeper in this network, not the {FormatJitter(asn.RawJitterMs)} a closer router reported. Traffic passes through the closer router to reach it, so the lower value is the true path jitter - the closer router just deprioritizes ICMP."; private static string JitterAssimilationTooltip(IspTargetHealth target) => - $"Showing {FormatJitter(target.P95JitterMs)} measured to a network reached through this router, not its own {FormatJitter(target.RawJitterMs)}. Traffic passes through it cleanly, so the higher reading is this router deprioritizing ICMP, not real jitter."; + $"Showing {FormatJitter(target.ScoredJitterMs)} measured to a network reached through this router, not its own {FormatJitter(target.RawJitterMs)}. Traffic passes through it cleanly, so the higher reading is this router deprioritizing ICMP, not real jitter."; private static string FormatRtt(double? ms) => ms.HasValue ? $"{ms.Value:0.00} ms" : "--"; @@ -1429,6 +1429,9 @@ else _ => "isp-score-poor" }; + // Appended to a score's class list (with a leading space) when it carries a caveat tooltip. + private static string CaveatClass(string? caveat) => caveat != null ? " isp-score-caveat" : ""; + private static string BarClass(int? score) => score switch { null => "isp-bar-none", diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs index 8711a36dd..f90577af6 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthModels.cs @@ -61,6 +61,10 @@ public class IspScoreFactor /// shows the fraction icon only when this is set. ///
public string? InvolvementTooltip { get; init; } + + /// For a reach-0 transit ASN scoring below 80: the score-text caveat about ICMP + /// deprioritization, copied from . Null otherwise. + public string? LowReachScoreCaveat { get; init; } } /// One of the three top-level score dimensions. @@ -96,7 +100,7 @@ public class IspAsnHealth public double? P95RttMs { get; init; } public double? MedianJitterMs { get; init; } - public double? P95JitterMs { get; init; } + public double? ScoredJitterMs { get; init; } /// True when the displayed jitter was assimilated from elsewhere (a cleaner /// farther cluster for transit, or the cleanest transit ASN for the ISP) rather than @@ -136,6 +140,15 @@ public class IspAsnHealth ? $"Carries {InvolvementReach} of {InvolvementHostTotal} internet targets (forward path), {w * 100:0}% weight" : $"Off the forward path; held at {w * 100:0}% (likely the return path from popular services)"; + /// Caveat for the score TEXT of a reach-0 transit ASN scoring below 80: nothing monitored + /// routes through it on the forward path, so a depressed score may be ICMP deprioritization at its + /// routers rather than real path trouble. Only for real transit ASNs (AsnNumber > 0) with known + /// attribution and zero forward-path reach; null otherwise. + public string? LowReachScoreCaveat => AsnNumber > 0 && ShowInvolvement && InvolvementReach == 0 + && OverallScore is int s && s < 80 + ? "Lightly weighted - nothing we are monitoring routes through this network; a low score is likely just ICMP deprioritization." + : null; + public int? LatencyStabilityScore { get; init; } public int? JitterScore { get; init; } public int? LossScore { get; init; } @@ -157,10 +170,10 @@ public class IspTargetHealth /// Displayed RTT: winsorized mean over the window. public double? RttMs { get; init; } - /// Effective (absolved) P95 jitter this hop is graded on. - public double? P95JitterMs { get; init; } + /// Effective (absolved) jitter this hop is graded on. + public double? ScoredJitterMs { get; init; } - /// This hop's own measured P95 jitter, before any absolve. + /// This hop's own measured jitter, before any absolve. public double? RawJitterMs { get; init; } /// True when a cleaner witness (transit/sibling/destination) pulled this hop's jitter below its own reading. diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthOptions.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthOptions.cs index 99b63e3ee..059dfa657 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthOptions.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthOptions.cs @@ -564,6 +564,15 @@ public class IspHealthOptions /// Weight of jitter in the per-ASN quality blend. public double AsnJitterWeight { get; set; } = 0.25; + /// + /// Percentile of a series' effective jitter used as its scored/displayed jitter and for the + /// path jitter floor, the absolve/witness comparisons, and the ISP/transit cap. P90 rather than + /// P95 so a link's harshest 5-10% of samples don't dominate the quality arm - intermittent bursts + /// are already caught by the separate congestion detector, so the quality arm reads the typical + /// tail, not the extreme. Scoring is floor-relative, so numerator and floor both use this value. + /// + public double AsnJitterScoringPercentile { get; set; } = 0.90; + /// Weight of packet loss in the per-ASN quality blend. public double AsnLossWeight { get; set; } = 0.2; diff --git a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs index 5f2128f0e..ff2850c02 100644 --- a/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs +++ b/src/NetworkOptimizer.Web/Services/Monitoring/IspHealth/IspHealthScorer.cs @@ -959,7 +959,7 @@ private IspAsnHealth GradeAsn( // The effective jitter: absolve-only across clusters (transit) or the ISP-wide // bound (ISP). This is what the card shows and what the ISP cap reads, so the // displayed value reflects the assimilation rather than the raw near hop. - P95JitterMs = effectiveJitter, + ScoredJitterMs = effectiveJitter, RttMadMs = mad, LossPct = losses.Count > 0 ? losses.Average() : null, ReachDeltaMs = reachDelta, @@ -975,7 +975,7 @@ private IspAsnHealth GradeAsn( }; } - /// The path jitter floor: the lowest scoring (P95) jitter across all ISP hops + /// The path jitter floor: the lowest scoring (P90) jitter across all ISP hops /// and transit clusters. Null when no series carries jitter. private double? ComputeJitterFloor(IspHealthInputs inputs) { @@ -995,15 +995,18 @@ void Add(IReadOnlyList samples) } /// - /// The jitter statistic used for scoring, the ISP/transit cap, and the cards: P95 of - /// the effective jitter. P95 (not median) because the tail is what the cards show and - /// what users reason about, and what hurts real-time traffic. The ISP and transit - /// jitter shown and scored are the same value. Null when none reported jitter. + /// The jitter statistic used for scoring, the ISP/transit cap, and the cards: the + /// (P90) of the effective jitter. + /// A tail percentile - not the median - because the tail is what the cards show, what users + /// reason about, and what hurts real-time traffic; P90 rather than P95 so a link's harshest + /// few percent of samples don't dominate the quality arm (intermittent bursts are caught by + /// the separate congestion detector). The ISP and transit jitter shown and scored are the same + /// value. Null when none reported jitter. /// - private static double? ScoringJitterOf(IReadOnlyList samples) + private double? ScoringJitterOf(IReadOnlyList samples) { var js = samples.Select(s => s.EffectiveJitterMs).Where(j => j.HasValue).Select(j => j!.Value).ToList(); - return js.Count > 0 ? SeriesStats.Percentile(js, 0.95) : null; + return js.Count > 0 ? SeriesStats.Percentile(js, _options.AsnJitterScoringPercentile) : null; } /// RTT stability ratio (MAD / median) of a sample set; lower is steadier. Null without RTT. @@ -1105,8 +1108,8 @@ private List GradeTransitAsns( .ToList() : new List<(List AncestorIps, double Jitter)>(); var transitWitnesses = baseGrades - .Where(b => b.Grade.P95JitterMs.HasValue) - .Select(b => (b.Series.AsnNumber, b.Series.AncestorIps, Jitter: b.Grade.P95JitterMs!.Value)) + .Where(b => b.Grade.ScoredJitterMs.HasValue) + .Select(b => (b.Series.AsnNumber, b.Series.AncestorIps, Jitter: b.Grade.ScoredJitterMs!.Value)) .ToList(); var grades = new List(); @@ -1173,7 +1176,7 @@ private List SelectPeeringReachedDestinations(IspHealthInputs inputs) /// jitter/loss/stability/reach basis as a real transit ASN, and the grades are AVERAGED - the series are /// never pooled. Pooling samples across targets at different RTT baselines (a flat ~2 ms peer beside a /// flat ~6 ms peer) manufactures cross-target variance that craters the stability sub-score, and lets a - /// single target's jitter tail dominate the pooled P95; per-peer grading keeps each target coherent and + /// single target's jitter tail dominate the pooled P90; per-peer grading keeps each target coherent and /// lets one degraded destination count only 1/N. Distance stays low (peered), so each reach ceiling /// barely bites and the grade is driven by jitter and loss. A proximity absolution then buys back a /// fraction of the jitter penalty when peering delivers the internet far closer than the transit @@ -1194,8 +1197,8 @@ private IspAsnHealth GradeIxPeering( foreach (var p in perPeer) { _logger?.LogDebug( - "ISP Health: IX Peering member {Name} -> {Overall} (stability {Stab}, jitter {Jit} @ P95 {P95j} ms, loss {Loss} @ {LossPct}%, reach ceiling {Reach})", - p.AsnName, p.OverallScore, p.LatencyStabilityScore, p.JitterScore, FormatMsOrNull(p.P95JitterMs), + "ISP Health: IX Peering member {Name} -> {Overall} (stability {Stab}, jitter {Jit} @ P{Pct} {Pj} ms, loss {Loss} @ {LossPct}%, reach ceiling {Reach})", + p.AsnName, p.OverallScore, p.LatencyStabilityScore, p.JitterScore, (int)Math.Round(_options.AsnJitterScoringPercentile * 100), FormatMsOrNull(p.ScoredJitterMs), p.LossScore, p.LossPct?.ToString("0.###", CultureInfo.InvariantCulture) ?? "n/a", p.ReachLatencyScore); } @@ -1246,7 +1249,7 @@ private IspAsnHealth GradeIxPeering( MeanRttMs = meanRtt, P95RttMs = AvgOf(p => p.P95RttMs), MedianJitterMs = AvgOf(p => p.MedianJitterMs), - P95JitterMs = AvgOf(p => p.P95JitterMs), + ScoredJitterMs = AvgOf(p => p.ScoredJitterMs), LossPct = AvgOf(p => p.LossPct), ReachDeltaMs = AvgOf(p => p.ReachDeltaMs), RawJitterMs = AvgOf(p => p.RawJitterMs), @@ -1275,9 +1278,9 @@ private List GradeIspHops( { // Transit witnesses: each transit ASN's ancestor IPs + its effective jitter. var transitJitterByAsn = transitAsns - .Where(a => a.P95JitterMs.HasValue) + .Where(a => a.ScoredJitterMs.HasValue) .GroupBy(a => a.AsnNumber) - .ToDictionary(g => g.Key, g => g.Min(a => a.P95JitterMs!.Value)); + .ToDictionary(g => g.Key, g => g.Min(a => a.ScoredJitterMs!.Value)); var transitWitnesses = transitSeries .Where(s => transitJitterByAsn.ContainsKey(s.AsnNumber)) .Select(s => (Ancestors: s.AncestorIps, Jitter: transitJitterByAsn[s.AsnNumber])) @@ -1341,7 +1344,7 @@ private List GradeIspHops( _logger?.LogDebug( "ISP Health: ISP hop {Target} (AS{Asn}) graded {Score} - measured jitter {Jitter} ms, effective {Eff} ms ({Witnesses} routes-through witnesses), reach +{Reach} ms", hop.TargetIds.FirstOrDefault(), hop.AsnNumber, grade.OverallScore, - FormatMsOrNull(measured), FormatMsOrNull(grade.P95JitterMs), witnesses.Count, FormatMsOrNull(grade.ReachDeltaMs)); + FormatMsOrNull(measured), FormatMsOrNull(grade.ScoredJitterMs), witnesses.Count, FormatMsOrNull(grade.ReachDeltaMs)); grades.Add(grade); } } @@ -1390,10 +1393,10 @@ private static List AggregateIspAsns(List hopGrades, && (e.TargetIds.Count == 0 || e.TargetIds.Any(t => targetSet.Contains(t)))) .ToList(); var means = hops.Select(h => h.MeanRttMs).Where(m => m.HasValue).Select(m => m!.Value).ToList(); - // Each hop's P95JitterMs is its per-hop effective (absolved) jitter; RawJitterMs + // Each hop's ScoredJitterMs is its per-hop effective (absolved) jitter; RawJitterMs // is its own measured reading. The card shows the mean effective and flags // assimilation when that fell below the mean measured. - var effJitters = hops.Select(h => h.P95JitterMs).Where(j => j.HasValue).Select(j => j!.Value).ToList(); + var effJitters = hops.Select(h => h.ScoredJitterMs).Where(j => j.HasValue).Select(j => j!.Value).ToList(); var rawJitters = hops.Select(h => h.RawJitterMs).Where(j => j.HasValue).Select(j => j!.Value).ToList(); double? effMean = effJitters.Count > 0 ? effJitters.Average() : null; double? rawMean = rawJitters.Count > 0 ? rawJitters.Average() : null; @@ -1410,7 +1413,7 @@ private static List AggregateIspAsns(List hopGrades, // RTT range across the ISP hops, on the same winsorized mean the hops display. MinRttMs = means.Count > 0 ? means.Min() : null, MaxRttMs = means.Count > 0 ? means.Max() : null, - P95JitterMs = effMean, + ScoredJitterMs = effMean, LossPct = lossVals.Count > 0 ? lossVals.Average() : null, OverallScore = scored.Count > 0 ? (int)Math.Round(scored.Average()) : null, CongestionEventCount = asnEvents.Count, @@ -1435,8 +1438,8 @@ private IspTargetHealth BuildIspTargetHealth(AsnSeries series, string? firstHopT var targetId = series.TargetIds.FirstOrDefault() ?? ""; var grade = hopGrades.FirstOrDefault(g => g.TargetIds.Contains(targetId)); // Jitter comes from the grade (the effective/absolved value the hop is scored on), so - // the row matches the grade beside it. Fall back to the hop's own raw P95 when ungraded. - var rawP95 = jitters.Count > 0 ? SeriesStats.Percentile(jitters, 0.95) : null; + // the row matches the grade beside it. Fall back to the hop's own raw jitter percentile when ungraded. + var rawScored = jitters.Count > 0 ? SeriesStats.Percentile(jitters, _options.AsnJitterScoringPercentile) : null; var isGraded = targetId == firstHopTargetId; var notTraced = notTracedTargetIds.Contains(targetId); return new IspTargetHealth @@ -1444,8 +1447,8 @@ private IspTargetHealth BuildIspTargetHealth(AsnSeries series, string? firstHopT TargetId = targetId, Name = series.AsnName ?? targetId, RttMs = SeriesStats.WinsorizedMean(rtts, winsorPercentile), - P95JitterMs = grade?.P95JitterMs ?? rawP95, - RawJitterMs = grade?.RawJitterMs ?? rawP95, + ScoredJitterMs = grade?.ScoredJitterMs ?? rawScored, + RawJitterMs = grade?.RawJitterMs ?? rawScored, JitterAssimilated = grade?.JitterAssimilated ?? false, LossPct = losses.Count > 0 ? losses.Average() : null, OverallScore = grade?.OverallScore, @@ -1524,7 +1527,8 @@ private static IspScoreDimension BuildAsnDimension(string name, double weight, L Description = a.CongestionEventCount > 0 ? $"{a.CongestionEventCount} congestion event{(a.CongestionEventCount == 1 ? "" : "s")} in the window." : null, - InvolvementTooltip = a.InvolvementTooltip + InvolvementTooltip = a.InvolvementTooltip, + LowReachScoreCaveat = a.LowReachScoreCaveat }).ToList(); var scored = asns.Where(a => a.OverallScore.HasValue).ToList(); diff --git a/src/NetworkOptimizer.Web/Services/UniFiConnectionService.cs b/src/NetworkOptimizer.Web/Services/UniFiConnectionService.cs index d338b03f1..9ceede77f 100644 --- a/src/NetworkOptimizer.Web/Services/UniFiConnectionService.cs +++ b/src/NetworkOptimizer.Web/Services/UniFiConnectionService.cs @@ -519,8 +519,13 @@ public async Task EnsureInitializedAsync() /// True when this site's console is reached through its agent tunnel and that tunnel /// isn't up yet - a transient "waiting for the agent" state, not a misconfiguration. /// The UI should prompt to wait/refresh rather than steering to connection setup. + /// Requires a configured, credentialed console: awaiting-agent is meaningless without a + /// saved target, so a half-configured site falls through to the "set up in Settings" banner + /// instead of showing the wait/refresh one (which would tell you to open Settings while only + /// offering Refresh). /// - public bool IsAwaitingAgent => _awaitingAgent && !_isConnected; + public bool IsAwaitingAgent => + _awaitingAgent && !_isConnected && _settings is { IsConfigured: true, HasCredentials: true }; public DateTime? LastConnectedAt => _lastConnectedAt; public bool IsUniFiOs => _client?.IsUniFiOs ?? false; diff --git a/src/NetworkOptimizer.Web/wwwroot/css/app.css b/src/NetworkOptimizer.Web/wwwroot/css/app.css index 190985171..a23e3f7c0 100644 --- a/src/NetworkOptimizer.Web/wwwroot/css/app.css +++ b/src/NetworkOptimizer.Web/wwwroot/css/app.css @@ -16804,6 +16804,14 @@ a.isp-health-hero-speed:hover { font-size: 0.9rem; font-weight: 700; text-align: right; + cursor: default; +} + +.isp-factor-score.isp-score-caveat, +.isp-asn-grade.isp-score-caveat { + cursor: help; + text-decoration: underline dotted; + text-underline-offset: 3px; } .isp-factor-description { @@ -16901,6 +16909,7 @@ a.isp-health-hero-speed:hover { .isp-asn-grade { font-size: 1.3rem; font-weight: 700; + cursor: default; } .isp-asn-stats { diff --git a/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs b/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs index fb72145d7..dcb90a343 100644 --- a/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs +++ b/tests/NetworkOptimizer.Web.Tests/IspHealth/IspHealthScorerTests.cs @@ -820,7 +820,7 @@ public void Transit_jitter_above_the_isp_mean_does_not_raise_isp_jitter() withJitteryTransit.IspAsnDimension.Score.Should().Be(noTransit.IspAsnDimension.Score, "a jittery transit ASN must not raise the ISP jitter (cap is min, not max)"); - withJitteryTransit.IspAsns.Single().P95JitterMs.Should().BeApproximately(0.3, 0.05, + withJitteryTransit.IspAsns.Single().ScoredJitterMs.Should().BeApproximately(0.3, 0.05, "the displayed ISP jitter stays at the ISP mean when transit is not cleaner"); } @@ -1005,7 +1005,7 @@ public void A_clean_farther_cluster_absolves_false_near_jitter() graded.JitterScore.Should().BeGreaterThan(ungraded.JitterScore!.Value, "the clean farther cluster disproves the near hop's false jitter"); graded.JitterScore.Should().BeGreaterThan(85); - graded.P95JitterMs.Should().BeApproximately(0.4, 0.1, + graded.ScoredJitterMs.Should().BeApproximately(0.4, 0.1, "the displayed jitter is the absolved value, not the near hop's 4 ms"); graded.JitterAssimilated.Should().BeTrue("the farther cluster pulled the jitter down"); graded.RawJitterMs.Should().BeApproximately(4.0, 0.1, "the raw near reading is kept for the tooltip"); diff --git a/tests/NetworkOptimizer.Web.Tests/IspHealth/JitterPercentileReplayTests.cs b/tests/NetworkOptimizer.Web.Tests/IspHealth/JitterPercentileReplayTests.cs new file mode 100644 index 000000000..2b49c1d47 --- /dev/null +++ b/tests/NetworkOptimizer.Web.Tests/IspHealth/JitterPercentileReplayTests.cs @@ -0,0 +1,99 @@ +using NetworkOptimizer.Web.Services.Monitoring.IspHealth; +using Xunit; +using Xunit.Abstractions; + +namespace NetworkOptimizer.Web.Tests.IspHealth; + +/// +/// Isolates the jitter-scoring lever (P95 -> P90 -> P50) over real exported latency data, to see +/// where a lower percentile's relief lands. Reuses and +/// the exact floor-relative jitter curve from IspHealthScorer.ScoreJitterVsFloor. Per target it +/// reports the raw jitter percentile and the resulting 0-100 jitter sub-score; the path jitter floor +/// tracks the chosen percentile (as ComputeJitterFloor does in production), so numerator and floor +/// move together. Gated on ISP_HEALTH_REPLAY_DIR - a no-op in CI and on other machines. +/// +public class JitterPercentileReplayTests +{ + private readonly ITestOutputHelper _output; + + public JitterPercentileReplayTests(ITestOutputHelper output) => _output = output; + + private static string? ReplayDir => Environment.GetEnvironmentVariable("ISP_HEALTH_REPLAY_DIR"); + + // Neutral (no per-tech band) floor-relative jitter curve, verbatim from IspHealthScorer.ScoreJitterVsFloor. + private static readonly IspHealthOptions Opt = new(); + + private static double JitterScore(double jitterMs, double floorMs) + { + var f = Math.Clamp(floorMs, Opt.JitterFloorMinMs, Opt.JitterFloorMaxMs); + return ScoreCurve.Interpolate(jitterMs, + (f, 100), (1.25 * f, 96), (1.5 * f, 91), (2.0 * f, 70), (5.0, 22), (12.0, 0)); + } + + // Matches SeriesStats.Percentile (linear, rank = p*(n-1)); inlined so the test needs no internals. + private static double Percentile(IReadOnlyList sortedAsc, double p) + { + var rank = p * (sortedAsc.Count - 1); + int lo = (int)Math.Floor(rank), hi = (int)Math.Ceiling(rank); + return lo == hi ? sortedAsc[lo] : sortedAsc[lo] + (sortedAsc[hi] - sortedAsc[lo]) * (rank - lo); + } + + private static string Band(double score) => + score >= 90 ? "Excellent" : score >= 75 ? "Good" : score >= 60 ? "Fair" : "Poor"; + + private static string Signed(double d) => (d >= 0 ? "+" : "-") + Math.Abs(d).ToString("0.0"); + + [Fact] + public void Replay_jitter_percentile_comparison() + { + if (ReplayDir == null) return; + foreach (var file in Directory.GetFiles(ReplayDir, "*.csv", SearchOption.AllDirectories).OrderBy(f => f)) + { + var header = File.ReadLines(file).FirstOrDefault(l => l.Contains("_time") && l.Contains("target_id")); + if (header == null || !header.Contains("jitter_ms")) continue; // only pivoted exports carry jitter + + var series = RealDataReplayTests.LoadSeries(file); + var targets = series + .Select(s => (s.AsnName, Jit: s.Samples + .Select(x => x.EffectiveJitterMs).Where(j => j.HasValue).Select(j => j!.Value) + .OrderBy(v => v).ToList())) + .Where(t => t.Jit.Count >= 20) + .Select(t => (t.AsnName, t.Jit, P50: Percentile(t.Jit, 0.50), P90: Percentile(t.Jit, 0.90), P95: Percentile(t.Jit, 0.95))) + .ToList(); + if (targets.Count == 0) continue; + + // Floor tracks the percentile, exactly as ComputeJitterFloor -> ScoringJitterOf does. + double floor95 = targets.Min(t => t.P95), floor90 = targets.Min(t => t.P90), floor50 = targets.Min(t => t.P50); + + var rows = targets.Select(t => new + { + t.AsnName, t.Jit.Count, t.P50, t.P90, t.P95, + S95 = JitterScore(t.P95, floor95), + S90 = JitterScore(t.P90, floor90), + S50 = JitterScore(t.P50, floor50) + }).OrderByDescending(r => r.S90 - r.S95).ToList(); + + _output.WriteLine($"=== {Path.GetFileName(file)} ({rows.Count} targets w/ jitter) ==="); + _output.WriteLine($"Path jitter floor: P95={floor95:0.00}ms P90={floor90:0.00}ms P50={floor50:0.00}ms"); + _output.WriteLine($"{"target",-30} {"n",5} {"P50",6} {"P90",6} {"P95",6} | {"jScore95",8} {"jScore90",8} {"jScore50",8} | {"dP90",6} {"dP50",6}"); + foreach (var r in rows) + { + _output.WriteLine( + $"{Trunc(r.AsnName, 30),-30} {r.Count,5} {r.P50,6:0.00} {r.P90,6:0.00} {r.P95,6:0.00} | " + + $"{r.S95,8:0.0} {r.S90,8:0.0} {r.S50,8:0.0} | {Signed(r.S90 - r.S95),6} {Signed(r.S50 - r.S95),6}"); + } + + int up90 = rows.Count(r => Band(r.S90) != Band(r.S95) && r.S90 > r.S95); + int down90 = rows.Count(r => Band(r.S90) != Band(r.S95) && r.S90 < r.S95); + int up50 = rows.Count(r => Band(r.S50) != Band(r.S95) && r.S50 > r.S95); + int down50 = rows.Count(r => Band(r.S50) != Band(r.S95) && r.S50 < r.S95); + _output.WriteLine($"Mean jitter score: P95={rows.Average(r => r.S95):0.0} P90={rows.Average(r => r.S90):0.0} P50={rows.Average(r => r.S50):0.0}"); + _output.WriteLine($"Band shifts vs P95 (Excellent>=90/Good>=75/Fair>=60/Poor): P90: +{up90} / -{down90} P50: +{up50} / -{down50}"); + _output.WriteLine($"Biggest P90 mover: {rows.First().AsnName} {rows.First().S95:0.0} -> {rows.First().S90:0.0} (P95 jit {rows.First().P95:0.00} -> P90 {rows.First().P90:0.00} ms)"); + _output.WriteLine($"Jitter is AsnJitterWeight={Opt.AsnJitterWeight:0.##} of the ASN quality blend, so +D jitter pts => ~+{Opt.AsnJitterWeight:0.##}*D on the grade (below the reach ceiling)."); + _output.WriteLine(""); + } + } + + private static string Trunc(string? s, int n) => (s ?? "") is { } v && v.Length > n ? v[..n] : s ?? ""; +} From 8f6ffba905d5262e92a91c67962d3cad20fdb22c Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 16:39:30 -0500 Subject: [PATCH 21/23] Multi-Site: LAN speed test toggle in settings-panel agent setup + reset wizard on site removal - The default-site "New Agent Token" flow hardcoded LanSpeedTest=false, so its generated install command could never include --lan-speed-test. Add a LAN speed test checkbox (mirroring the wizard) that live-updates the command; non-default sites already prompt via the wizard. - After removing a site, reset the Add-a-Site wizard's completed screen (same ResetIfComplete used when opening a site's Configuration) so it doesn't linger. --- .../Components/Pages/Settings.razor | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index a00d9ab38..41924dcaf 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -3261,7 +3261,13 @@ } @if (_newAgentToken != null && _newAgentTokenSiteId == site.Id) { - + + } @@ -3603,6 +3609,9 @@ return; } _expandedSiteId = null; + // Clear the Add-a-Site wizard's completed screen too, same as opening a + // site's Configuration does, so it doesn't linger after a removal. + _setupWizard?.ResetIfComplete(); _multiSiteMessage = $"Site \"{site.Name}\" was removed."; _multiSiteMessageClass = "success"; await RefreshSites(); @@ -3670,6 +3679,7 @@ private bool _creatingAgent; private string? _newAgentToken; private int? _newAgentTokenSiteId; + private bool _newAgentLanSpeedTest; private SiteSetupWizard? _setupWizard; // A site still needs agent setup until one of its agents has enrolled; @@ -3901,6 +3911,7 @@ var (agent, token) = await AgentEnrollment.CreateAgentAsync(site.Id, NextAgentName(site.Id)); _newAgentToken = token; _newAgentTokenSiteId = site.Id; + _newAgentLanSpeedTest = false; await LoadAgentsAsync(); } finally From b91e2dfb127e31d659eec87dec3303fabe9544e5 Mon Sep 17 00:00:00 2001 From: TJ da Tuna Date: Mon, 20 Jul 2026 16:46:29 -0500 Subject: [PATCH 22/23] Multi-Site: reset the Add-a-Site wizard unconditionally on site removal Removal used ResetIfComplete (step-4 only), but you can delete a site while the wizard is on the agent step (step 3) for that very site. Add a public Reset() that clears the form from any step and use it on removal; opening a site's Configuration keeps the conditional reset so mid-add input isn't discarded. --- .../Components/Pages/Settings.razor | 6 +++--- .../Components/Shared/SiteSetupWizard.razor | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor index 41924dcaf..ace19e36b 100644 --- a/src/NetworkOptimizer.Web/Components/Pages/Settings.razor +++ b/src/NetworkOptimizer.Web/Components/Pages/Settings.razor @@ -3609,9 +3609,9 @@ return; } _expandedSiteId = null; - // Clear the Add-a-Site wizard's completed screen too, same as opening a - // site's Configuration does, so it doesn't linger after a removal. - _setupWizard?.ResetIfComplete(); + // Always reset the Add-a-Site wizard after a removal - unconditionally, since + // it may be sitting on the agent step for the very site just deleted. + _setupWizard?.Reset(); _multiSiteMessage = $"Site \"{site.Name}\" was removed."; _multiSiteMessageClass = "success"; await RefreshSites(); diff --git a/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor b/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor index 35fb0a4cc..3aeba2629 100644 --- a/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor +++ b/src/NetworkOptimizer.Web/Components/Shared/SiteSetupWizard.razor @@ -465,6 +465,17 @@ StateHasChanged(); } + /// + /// Unconditionally returns the wizard to a fresh Add-a-Site form, whatever step + /// it is on. Used when the site it may be operating on is removed - e.g. deleting + /// a site while its agent-setup step is showing. + /// + public void Reset() + { + ResetForm(); + StateHasChanged(); + } + private void ResetForm() { _step = 1; From 914888b1066f5e4e8b154d653625326d8d3897be Mon Sep 17 00:00:00 2001 From: "TJ @ Ozark Connect" <109822114+tvancott42@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:47:34 -0500 Subject: [PATCH 23/23] On-site agent installer: AppArmor-aware LAN speed test, clean uninstall, tidier output (#1035) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * agent installer: auto-remediate AppArmor-confined nginx for LAN speed test Some distros ship an enforcing AppArmor profile on the nginx binary that only permits nginx's stock paths, so our dedicated speed test master - which keeps its pid, logs, and webroot under the install dir - is denied even as root (a MAC denial, not file perms), and the LAN speed test never serves. The installer now, ONLY after `nginx -t` has already failed AND the kernel log shows AppArmor denying our install dir, adds a local, additive override under /etc/apparmor.d/local granting the confined nginx access to the install dir, reloads just that one profile, and re-tests. It never edits the vendor profile; a parse error makes apparmor_parser abort and keep the running profile, so a bad override can't unconfine the host's own nginx. If the profile can't be found or doesn't consume the local include, the override is inert and we fall back to a precise manual hint (local override, or `aa-complain`). Reactive by design: on any host where `nginx -t` already passes, none of this code is reached, so currently-working installs are byte-for-byte unchanged. The speed test failure remains non-fatal - the agent itself is unaffected. * agent installer: correct AppArmor fix hint for non-standard nginx profiles The fallback hint hardcoded the profile filename (usr.sbin.nginx) and pointed aa-complain at the resolved binary, both wrong when the confining profile is named differently (e.g. /usr/bin/nginx) or isn't file-backed under /etc/apparmor.d (snap/custom profiles - Mother's case). Now derive the real profile name from the denial record, search the snap profile dir as well as /etc/apparmor.d, and only attempt the local-include override when the profile is actually a file under /etc/apparmor.d (the only place local/ is consumed). When it isn't, the hint leads with `aa-complain ''` instead of a bogus local-override path. * agent installer: --uninstall, root-worker for non-traversable dirs, safe override - Add `--uninstall` to the bare-metal installer (parity with the gateway one, and the README already implied it existed). Stops both services BEFORE deleting their unit files - so it never leaves a running "ghost" unit like a manual rm does - then removes the units, enable symlink, our AppArmor override, and the install dir. Leaves the host's own nginx and AppArmor profile untouched. Documented in the agent README. - Fix LAN speed test 403 when the install dir isn't world-traversable (e.g. under /root, mode 700): unprivileged nginx workers can't reach the webroot. Detect a non-traversable ancestor chain (handling the sticky bit, so /tmp is fine) and run the workers as root only then; the /opt default keeps the safer unprivileged user. - Make the AppArmor local override additive and marker-bracketed instead of clobbering the whole /etc/apparmor.d/local file, so a user's own rules there survive and --uninstall can remove exactly our block. * agent installer: find nginx AppArmor profiles under /usr/share/apparmor.d too Some distros ship the nginx AppArmor profile under /usr/share/apparmor.d (with /etc/apparmor.d reserved for local overrides) rather than in /etc/apparmor.d, so the auto-fix couldn't find it and fell back to the hint. Search that dir as well, and attempt the local override for a profile found there (the override still lives under /etc/apparmor.d/local, the standard include base; we reload the profile source wherever it ships). The nginx -t re-test still gates success, so if the profile has no local/ hook we fall back to the (now simpler) complain-mode hint. * agent installer: don't match/reload AppArmor cache files; skip cache on reload (-rT) Mother's nginx profile ships no findable source file - only a compiled cache at /usr/share/apparmor.d/cache/usr.bin.nginx. The broadened search matched that cache and apparmor_parser -r on a binary cache produced the confusing 'reload failed'. Exclude cache/ (and local/) from the profile search so an unfindable source now falls straight to the clear aa-complain hint, and reload with -rT so a read-only cache dir can't fail the reload on appliance distros. * agent installer: detect aa-complain in the hint; scope uninstall's summary - AppArmor hint now detects aa-complain: shows it when apparmor-utils is present, otherwise the tool-free, source-free securityfs unload (echo profile > .remove) - which works on appliance distros that ship neither aa-complain nor a profile source file (the Mother case). - --uninstall summary only mentions nginx/AppArmor when they were actually involved: a monitoring-only install (no --lan-speed-test) never touches nginx, so it just says "agent removed"; a speed-test install says nginx is untouched; and it notes the AppArmor override only when one was actually removed. * agent installer: cleaner, structured output Silence the curl progress tables (downloads are fast; -fsSL), and add colorized, sectioned output - a bold title, per-section dividers + "==>" headers, "✓" for successes, "⚠" for warnings, dim detail lines. Colors/markers collapse to plain text when stdout isn't a terminal, so piped/logged output stays clean. Suppress the systemd "Created symlink" noise with `systemctl enable --quiet`. Same for the --uninstall teardown output. * agent installer: AppArmor hint gives persistent guidance only Drop the temporary securityfs unload (echo profile > .remove) from the fallback hint - it evaporates on reboot, which is useless in a production deployment. The hint now frames the speed test as optional (agent + monitoring unaffected) and points to persistent remediation only: aa-complain (edits+reloads the profile, so it survives reboot) when apparmor-utils is present, otherwise a persistent policy exception added by the host admin. * agent installer: AppArmor changes are opt-in (--configure-apparmor) The installer no longer modifies the host's AppArmor policy by default - an app installer silently editing a security team's policy is a non-starter for enterprise deployments. The scoped local override now only runs under the new --configure-apparmor flag. Default path: detect, stay non-fatal (agent + monitoring unaffected), and the guidance tells the operator to re-run the install command with --configure-apparmor to add a persistent, scoped exception. When the flag IS given but the host ships a source-less / hook-less profile (no scoped exception is possible), the guidance says so plainly and defers to the admin rather than pretending or hacking. Documented in the agent README. * agent installer: make it obvious nginx is only for the LAN speed test Clarify in --lan-speed-test / --configure-apparmor help and the README that nginx is used solely to serve the speed test - core monitoring never touches it - so a monitoring-only user doesn't read the AppArmor/nginx handling as something central. * agent installer: trim the nginx/apparmor help + docs wording * agent installer: sudo in the nginx diagnose hint (runs as non-root later) * agent installer: spruce gateway + docker installer output too Same structured, colorized output as install-native (title, section dividers + "==>" headers, "✓"/"⚠", dim detail; collapses to plain text when not a TTY). Output-only: no logic, conditions, curl targets, compose, or systemctl behavior changed. Gateway also silences the binary download meter (-fsSL) and the systemd "Created symlink" line (enable --quiet; verified accepted on systemd 247). * agent README: fix flag scoping + note AppArmor on the speed test - --uninstall and --configure-apparmor are bare-metal-only; they were listed under "Both scripts accept" but the Docker installer has neither. Split them out. - Add a short note in "LAN speed test serving" on AppArmor-confined nginx: the page may be blocked (monitoring unaffected), re-run with --configure-apparmor for a scoped exception, else an admin grants it. --- scripts/agent/install-agent-gateway.sh | 59 +++-- scripts/agent/install-docker.sh | 38 +++- scripts/agent/install-native.sh | 301 +++++++++++++++++++++++-- src/NetworkOptimizer.Agent/README.md | 23 +- 4 files changed, 365 insertions(+), 56 deletions(-) diff --git a/scripts/agent/install-agent-gateway.sh b/scripts/agent/install-agent-gateway.sh index 4b46b8238..4ebc086a0 100755 --- a/scripts/agent/install-agent-gateway.sh +++ b/scripts/agent/install-agent-gateway.sh @@ -59,19 +59,34 @@ while [ $# -gt 0 ]; do esac done -err() { echo "Error: $*" >&2; exit 1; } +# --- Output helpers ----------------------------------------------------------- +# Colorized, structured output; colors collapse to empty when stdout isn't a +# terminal, so piped/logged output stays clean. +if [ -t 1 ]; then + _b=$'\e[1m'; _dim=$'\e[2m'; _grn=$'\e[32m'; _ylw=$'\e[33m'; _red=$'\e[31m'; _cyn=$'\e[36m'; _rst=$'\e[0m' +else + _b=; _dim=; _grn=; _ylw=; _red=; _cyn=; _rst= +fi +_rule="$(printf '\xe2\x94\x80%.0s' {1..52})" # ── divider between sections +step() { printf '\n%s%s%s\n%s==>%s %s%s%s\n' "$_dim" "$_rule" "$_rst" "${_cyn}${_b}" "$_rst" "$_b" "$*" "$_rst"; } +ok() { printf ' %s\xe2\x9c\x93%s %s\n' "$_grn" "$_rst" "$*"; } +note() { printf ' %s%s%s\n' "$_dim" "$*" "$_rst"; } +warn() { printf ' %s\xe2\x9a\xa0%s %s\n' "$_ylw" "$_rst" "$*"; } +err() { printf '%sError:%s %s\n' "${_red}${_b}" "$_rst" "$*" >&2; exit 1; } [ "$(id -u)" -eq 0 ] || err "Run as root (the gateway's default SSH user is root)." command -v systemctl >/dev/null 2>&1 || err "systemd is required (systemctl not found)." # --- Teardown -------------------------------------------------------------- if [ "$UNINSTALL" = true ]; then - echo "Removing ${SERVICE_NAME} and ${INSTALL_DIR}..." + step "Removing the Network Optimizer agent" + note "${SERVICE_NAME} + ${INSTALL_DIR}" systemctl disable --now "${SERVICE_NAME}.service" 2>/dev/null || true rm -f "/etc/systemd/system/${SERVICE_NAME}.service" systemctl daemon-reload 2>/dev/null || true rm -rf "$INSTALL_DIR" - echo "Done - the gateway is back to stock." + ok "Removed - the gateway is back to stock." + printf '\n' exit 0 fi @@ -96,36 +111,39 @@ esac # is already accounted for in MemAvailable). MIN_AVAILABLE_MB=256 if ! systemctl is-active --quiet "${SERVICE_NAME}.service"; then + step "Memory pre-flight" AVAILABLE_MB="$(awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo)" if [ -z "$AVAILABLE_MB" ]; then - echo "Warning: could not read MemAvailable from /proc/meminfo - skipping the memory check." >&2 + warn "could not read MemAvailable from /proc/meminfo - skipping the memory check." elif [ "$AVAILABLE_MB" -lt "$MIN_AVAILABLE_MB" ]; then err "only ${AVAILABLE_MB} MB of memory is available; the agent needs ${MIN_AVAILABLE_MB} MB of headroom so it stays well clear of routing/IPS. Free up memory (e.g. remove unused UniFi applications) or run the agent on a separate box (see install-native.sh)." else - echo "Memory check: ${AVAILABLE_MB} MB available (need ${MIN_AVAILABLE_MB} MB) - OK" + ok "${AVAILABLE_MB} MB available (need ${MIN_AVAILABLE_MB} MB)" fi fi -echo "Installing Network Optimizer agent to ${INSTALL_DIR} (${RID}, monitoring-only)" +printf '\n%sNetwork Optimizer on-site agent (gateway, monitoring-only)%s\n' "$_b" "$_rst" +note "Installing to ${INSTALL_DIR} (${RID})" mkdir -p "$INSTALL_DIR" # Download to a temp name and rename into place: writing over the binary while # the agent is running fails with ETXTBSY, but rename swaps the directory entry # and the running process keeps its old inode until the restart below. -echo "Downloading agent binary..." -curl -fSL "${RELEASE_BASE}/NetworkOptimizer.Agent-${RID}" -o "${INSTALL_DIR}/NetworkOptimizer.Agent.new" +step "Downloading agent binary" +curl -fsSL "${RELEASE_BASE}/NetworkOptimizer.Agent-${RID}" -o "${INSTALL_DIR}/NetworkOptimizer.Agent.new" chmod +x "${INSTALL_DIR}/NetworkOptimizer.Agent.new" mv -f "${INSTALL_DIR}/NetworkOptimizer.Agent.new" "${INSTALL_DIR}/NetworkOptimizer.Agent" +ok "agent (${RID})" CONFIG="${INSTALL_DIR}/agent.json" +step "Configuring the agent" # Preserve an already-enrolled config so re-running to update the binary never # wipes the persisted key. if grep -q '"agentKey"' "$CONFIG" 2>/dev/null; then - echo "Existing enrolled agent config found - keeping it." + note "Existing enrollment found - keeping agent.json" else [ -n "$TOKEN" ] || err "--token is required for a first-time install." - echo "Writing ${CONFIG}" { echo "{" echo " \"serverUrl\": \"${SERVER%/}\"," @@ -135,9 +153,10 @@ else echo "}" } > "$CONFIG" chmod 600 "$CONFIG" + ok "Wrote ${CONFIG}" fi -echo "Installing ${SERVICE_NAME}.service" +step "Installing the agent service" cat > "/etc/systemd/system/${SERVICE_NAME}.service" <&2; exit 1; } +# --- Output helpers ----------------------------------------------------------- +# Colorized, structured output; colors collapse to empty when stdout isn't a +# terminal, so piped/logged output stays clean. +if [ -t 1 ]; then + _b=$'\e[1m'; _dim=$'\e[2m'; _grn=$'\e[32m'; _ylw=$'\e[33m'; _red=$'\e[31m'; _cyn=$'\e[36m'; _rst=$'\e[0m' +else + _b=; _dim=; _grn=; _ylw=; _red=; _cyn=; _rst= +fi +_rule="$(printf '\xe2\x94\x80%.0s' {1..52})" # ── divider between sections +step() { printf '\n%s%s%s\n%s==>%s %s%s%s\n' "$_dim" "$_rule" "$_rst" "${_cyn}${_b}" "$_rst" "$_b" "$*" "$_rst"; } +ok() { printf ' %s\xe2\x9c\x93%s %s\n' "$_grn" "$_rst" "$*"; } +note() { printf ' %s%s%s\n' "$_dim" "$*" "$_rst"; } +warn() { printf ' %s\xe2\x9a\xa0%s %s\n' "$_ylw" "$_rst" "$*"; } +err() { printf '%sError:%s %s\n' "${_red}${_b}" "$_rst" "$*" >&2; exit 1; } [ -n "$SERVER" ] || err "--server is required (the central server's HTTPS address)" case "$SERVER" in @@ -61,21 +74,23 @@ if [ "$(id -u)" -ne 0 ]; then SUDO="sudo" fi -echo "Installing Network Optimizer agent to ${INSTALL_DIR}" +printf '\n%sNetwork Optimizer on-site agent (Docker)%s\n' "$_b" "$_rst" +note "Installing to ${INSTALL_DIR}" $SUDO mkdir -p "${INSTALL_DIR}/data" -# Compose template +step "Fetching the compose template" $SUDO curl -fsSL "$COMPOSE_URL" -o "${INSTALL_DIR}/docker-compose.yml" +ok "docker-compose.yml" CONFIG="${INSTALL_DIR}/data/agent.json" +step "Configuring the agent" # Preserve an already-enrolled config so re-running the installer (e.g. to # update the image) never wipes the persisted agent key. if $SUDO grep -q '"agentKey"' "$CONFIG" 2>/dev/null; then - echo "Existing enrolled agent config found - keeping it." + note "Existing enrollment found - keeping agent.json" else [ -n "$TOKEN" ] || err "--token is required for a first-time install" - echo "Writing ${CONFIG}" TMP_CONFIG="$(mktemp)" { echo "{" @@ -90,13 +105,16 @@ else } > "$TMP_CONFIG" $SUDO cp "$TMP_CONFIG" "$CONFIG" rm -f "$TMP_CONFIG" + ok "Wrote ${CONFIG}" fi -echo "Starting agent..." +step "Starting the agent" $SUDO $COMPOSE -f "${INSTALL_DIR}/docker-compose.yml" pull $SUDO $COMPOSE -f "${INSTALL_DIR}/docker-compose.yml" up -d +ok "container up" -echo -echo "Agent started. It enrolls, then holds a tunnel to ${SERVER%/}." -echo "Watch it come Online in the web UI, or follow logs:" -echo " ${SUDO} docker logs -f network-optimizer-agent" +step "Done" +ok "Agent started" +note "It enrolls, then holds a tunnel to ${SERVER%/} - watch it come Online in the web UI." +note "Logs: ${SUDO} docker logs -f network-optimizer-agent" +printf '\n' diff --git a/scripts/agent/install-native.sh b/scripts/agent/install-native.sh index 3956680fb..a2159be06 100755 --- a/scripts/agent/install-native.sh +++ b/scripts/agent/install-native.sh @@ -17,6 +17,10 @@ # --lan-speed-test Host the LAN speed test page (port 3000) and iperf3 (5201) # --insecure Accept a self-signed cert on the server's reverse proxy # --dir PATH Install directory (default: /opt/netopt-agent) +# --uninstall Stop and remove the agent, its services, install dir, and any +# AppArmor override this installer added, then exit +# --configure-apparmor With --lan-speed-test: add a persistent AppArmor exception +# if the host's nginx profile blocks the speed test (off by default) set -euo pipefail @@ -24,8 +28,11 @@ SERVER="" TOKEN="" LAN_SPEED_TEST=false INSECURE=false +UNINSTALL=false +CONFIGURE_APPARMOR=false INSTALL_DIR="/opt/netopt-agent" SERVICE_NAME="netopt-agent" +SPEEDTEST_SERVICE="netopt-speedtest-nginx" RELEASE_BASE="https://github.com/Ozark-Connect/NetworkOptimizer/releases/latest/download" while [ $# -gt 0 ]; do @@ -34,20 +41,241 @@ while [ $# -gt 0 ]; do --token) TOKEN="$2"; shift 2 ;; --lan-speed-test) LAN_SPEED_TEST=true; shift ;; --insecure) INSECURE=true; shift ;; + --uninstall) UNINSTALL=true; shift ;; + --configure-apparmor) CONFIGURE_APPARMOR=true; shift ;; --dir) INSTALL_DIR="$2"; shift 2 ;; *) echo "Unknown option: $1" >&2; exit 1 ;; esac done -err() { echo "Error: $*" >&2; exit 1; } +# --- Output helpers ----------------------------------------------------------- +# Colorized, structured output; colors collapse to empty when stdout isn't a +# terminal (piped/redirected/logged), so captured output stays clean. +if [ -t 1 ]; then + _b=$'\e[1m'; _dim=$'\e[2m'; _grn=$'\e[32m'; _ylw=$'\e[33m'; _red=$'\e[31m'; _cyn=$'\e[36m'; _rst=$'\e[0m' +else + _b=; _dim=; _grn=; _ylw=; _red=; _cyn=; _rst= +fi +_rule="$(printf '\xe2\x94\x80%.0s' {1..52})" # ── divider between sections +step() { printf '\n%s%s%s\n%s==>%s %s%s%s\n' "$_dim" "$_rule" "$_rst" "${_cyn}${_b}" "$_rst" "$_b" "$*" "$_rst"; } +ok() { printf ' %s\xe2\x9c\x93%s %s\n' "$_grn" "$_rst" "$*"; } +note() { printf ' %s%s%s\n' "$_dim" "$*" "$_rst"; } +warn() { printf ' %s\xe2\x9a\xa0%s %s\n' "$_ylw" "$_rst" "$*"; } +err() { printf '%sError:%s %s\n' "${_red}${_b}" "$_rst" "$*" >&2; exit 1; } + +# --- LAN speed test nginx: AppArmor remediation ------------------------------- +# Some distros ship an ENFORCING AppArmor profile on the nginx binary that only +# permits nginx's stock paths (/var/log/nginx, /run/nginx.pid, ...). Our dedicated +# speed test master keeps its pid, logs, and webroot under the install dir, which +# such a profile denies even to root - the failure is a MAC denial, not file +# permissions. Everything below runs ONLY after `nginx -t` has already failed and +# only acts on a real AppArmor denial of our install dir, so any host where the +# speed test already works never enters this path. + +# Markers bracketing our block inside an AppArmor local override, so it can be added +# without clobbering a user's existing file and removed cleanly on --uninstall. +AA_MARK_BEGIN="# NETOPT-AGENT-OVERRIDE BEGIN" +AA_MARK_END="# NETOPT-AGENT-OVERRIDE END" + +# True when the kernel log shows AppArmor denying access to a path under the +# install dir (recorded when the failing `nginx -t` ran). This is the trigger: +# without an actual denial we leave AppArmor entirely alone. +_aa_denied_install_dir() { + [ -d /sys/kernel/security/apparmor ] || return 1 + { journalctl -k -n 400 --no-pager 2>/dev/null || dmesg 2>/dev/null || true; } \ + | grep -q "apparmor=\"DENIED\".*name=\"${INSTALL_DIR}" 2>/dev/null +} + +# The AppArmor profile NAME confining nginx (e.g. "/usr/bin/nginx"), read from the +# denial record. Empty if none was logged. +_aa_nginx_profile_name() { + { journalctl -k -n 400 --no-pager 2>/dev/null || dmesg 2>/dev/null || true; } \ + | grep "apparmor=\"DENIED\".*name=\"${INSTALL_DIR}" \ + | grep -o 'profile="[^"]*"' | head -1 | sed 's/^profile="//; s/"$//' || true +} + +# AppArmor's conventional on-disk filename for a profile name: drop the leading +# '/', turn the remaining '/' into '.' - e.g. /usr/bin/nginx -> usr.bin.nginx. +_aa_profile_filename() { local n="${1#/}"; printf '%s' "${n//\//.}"; } + +# Path of the vendor AppArmor profile file confining nginx: the conventionally-named +# file first, else a content match, across the standard profile dirs. Empty when the +# profile isn't file-backed there (e.g. a snap profile) - then a local/ override +# can't apply and we fall back to the hint. Cached after the first lookup. +AA_NGINX_PROFILE_FILE="" +AA_NGINX_PROFILE_FILE_SET="" +_aa_nginx_profile_file() { + [ -n "$AA_NGINX_PROFILE_FILE_SET" ] && { printf '%s' "$AA_NGINX_PROFILE_FILE"; return 0; } + AA_NGINX_PROFILE_FILE_SET=1 + local pname fname dir cand file="" + pname="$(_aa_nginx_profile_name)"; [ -n "$pname" ] || pname="$NGINX_BIN" + fname="$(_aa_profile_filename "$pname")" + for dir in /etc/apparmor.d /usr/share/apparmor.d /var/lib/snapd/apparmor/profiles; do + [ -d "$dir" ] || continue + if [ -f "$dir/$fname" ]; then + cand="$dir/$fname" + else + # Exclude local/ (include-only) and cache/ (compiled binaries, not loadable + # profile sources - matching one leads to a bogus apparmor_parser reload). + cand="$(grep -rslF "$pname" "$dir" 2>/dev/null | grep -vE '/(local|cache)/' | head -1 || true)" + fi + [ -n "$cand" ] && { file="$cand"; break; } + done + AA_NGINX_PROFILE_FILE="$file" + printf '%s' "$file" +} + +# Additive, local-only AppArmor override letting the confined nginx use the install +# dir. OPT-IN: only runs under --configure-apparmor, so the installer never modifies +# host security policy unless explicitly asked. The override is scoped and persistent +# (a file under /etc/apparmor.d/local); never edits the vendor profile; reloads only +# that one profile. A parse error makes apparmor_parser abort and keep the running +# profile, so a bad override can't unconfine the host's own nginx. Caller re-tests +# `nginx -t` to judge the result. +maybe_fix_apparmor_nginx() { + [ "$CONFIGURE_APPARMOR" = true ] || return 0 + command -v apparmor_parser >/dev/null 2>&1 || return 0 + _aa_denied_install_dir || return 0 + + local file base + file="$(_aa_nginx_profile_file)" + case "$file" in + # Vendor profiles under these dirs use the standard local/ include base + # (/etc/apparmor.d/local), so an override there applies. The override file + # always lives under /etc/apparmor.d/local regardless of where the profile + # itself ships; we reload the profile source wherever it was found. + /etc/apparmor.d/*|/usr/share/apparmor.d/*) ;; + # Snap or otherwise non-standard profile won't consume /etc/apparmor.d/local: + # a local override wouldn't apply. Leave it to the hint below. + *) return 0 ;; + esac + base="$(basename "$file")" + local localfile="/etc/apparmor.d/local/${base}" + + echo "AppArmor is blocking ${INSTALL_DIR}; adding a local override (${localfile}) and reloading." + mkdir -p /etc/apparmor.d/local + # Additive and idempotent: strip any prior block of ours (e.g. an earlier install + # to a different dir), then append the current one. Never clobbers a user's own + # rules in this file. + [ -f "$localfile" ] && sed -i "/${AA_MARK_BEGIN}/,/${AA_MARK_END}/d" "$localfile" + { + echo "$AA_MARK_BEGIN" + echo "# Added by the Network Optimizer agent installer - removed by --uninstall." + echo "# Lets the LAN speed test nginx master use its pid, logs, and webroot here." + echo "${INSTALL_DIR}/ r," + echo "${INSTALL_DIR}/** rw," + echo "$AA_MARK_END" + } >> "$localfile" + + # -T skips the compiled cache, so a read-only cache dir (common on appliance + # distros) doesn't fail the reload; the profile still loads from source. + apparmor_parser -rT "$file" >/dev/null 2>&1 \ + || echo " apparmor_parser reload failed; the override may not have taken (the profile may not include local/${base})." + return 0 +} + +# Actionable manual remediation, printed only when the speed test nginx still fails +# because of an AppArmor denial (so it never nags on unrelated failures). By the time +# this prints, the automatic local-override fix has already been tried or wasn't +# possible, so complain mode - which works regardless of how the profile is shipped - +# is the reliable recommendation. Names the real profile. +print_speedtest_apparmor_hint() { + _aa_denied_install_dir || return 0 + local pname + pname="$(_aa_nginx_profile_name)"; [ -n "$pname" ] || pname="$NGINX_BIN" + echo " AppArmor profile '${pname}' denies nginx access to ${INSTALL_DIR}." + echo " The agent and monitoring are unaffected - this gates only the optional speed test page." + if [ "$CONFIGURE_APPARMOR" != true ]; then + echo " To have the installer add a persistent, scoped AppArmor exception, re-run the" + echo " install command with --configure-apparmor added." + else + echo " A scoped exception couldn't be applied here: this host's nginx profile has no" + echo " source file or local/ include hook to attach one to. Enabling the speed test" + echo " needs a persistent change to that profile by your admin, or serving it from a" + echo " host whose nginx isn't AppArmor-confined." + fi +} + +# Remove any AppArmor local override this installer added (marker-guarded, so a +# user's own rules in the same file are preserved), then best-effort reload so the +# removal takes effect. Called from teardown. +remove_apparmor_override() { + local f changed=0 + for f in /etc/apparmor.d/local/*; do + [ -f "$f" ] || continue + grep -q "$AA_MARK_BEGIN" "$f" 2>/dev/null || continue + sed -i "/${AA_MARK_BEGIN}/,/${AA_MARK_END}/d" "$f" + [ -s "$f" ] || rm -f "$f" # nothing left but our block -> drop the file + changed=1 + note "Removed AppArmor override from ${f}" + done + [ "$changed" = 1 ] && systemctl reload apparmor >/dev/null 2>&1 || true + AA_OVERRIDE_REMOVED="$changed" + return 0 +} + +# nginx workers drop privileges after start, so an install dir whose ancestors are +# not world-traversable (e.g. under /root, mode 700) makes them 403 - they can't +# reach the webroot. True in that case, so the wrapper should run workers as root. +_webroot_needs_root_worker() { + local p mode + p="$(readlink -f "$1" 2>/dev/null || echo "$1")" + while [ -n "$p" ] && [ "$p" != "/" ]; do + mode="$(stat -c '%A' "$p" 2>/dev/null || echo '')" + case "$mode" in + "") ;; # stat failed at this level; ignore + *x|*t) ;; # other-execute set (x, or t with the sticky bit as on + # /tmp) -> traversable, keep walking up + *) return 0 ;; # last char '-' or 'T' -> blocks an unprivileged worker + esac + p="$(dirname "$p")" + done + return 1 +} + +# Stop and remove everything this installer created, in an order that never leaves a +# running "ghost" unit (services are stopped before their unit files are deleted). +# Leaves the host's own nginx untouched; removes only the AppArmor override this +# installer itself added (if any), reporting accordingly. +uninstall_agent() { + step "Removing the Network Optimizer agent" + note "${SERVICE_NAME} + ${INSTALL_DIR}" + # Whether this install ever set up the speed test (and thus borrowed nginx) - so + # the summary only mentions nginx/AppArmor when they were actually involved. + local had_speedtest=0 + [ -f "/etc/systemd/system/${SPEEDTEST_SERVICE}.service" ] && had_speedtest=1 + systemctl disable --now "${SERVICE_NAME}.service" "${SPEEDTEST_SERVICE}.service" 2>/dev/null || true + rm -f "/etc/systemd/system/${SERVICE_NAME}.service" \ + "/etc/systemd/system/${SPEEDTEST_SERVICE}.service" \ + "/etc/systemd/system/multi-user.target.wants/${SERVICE_NAME}.service" + systemctl daemon-reload 2>/dev/null || true + systemctl reset-failed "${SERVICE_NAME}.service" "${SPEEDTEST_SERVICE}.service" 2>/dev/null || true + remove_apparmor_override + rm -rf "$INSTALL_DIR" + if [ "${AA_OVERRIDE_REMOVED:-0}" = 1 ]; then + ok "Agent removed, including the AppArmor override it had added. The host's own nginx is untouched." + elif [ "$had_speedtest" = 1 ]; then + ok "Agent removed. The host's own nginx is untouched." + else + ok "Agent removed." + fi + printf '\n' +} + +[ "$(id -u)" -eq 0 ] || err "Run as root (needed to manage the systemd service): sudo bash install-native.sh ..." +command -v systemctl >/dev/null 2>&1 || err "systemd is required (systemctl not found)" + +# Teardown short-circuits the install: needs neither --server nor a token. +if [ "$UNINSTALL" = true ]; then + uninstall_agent + exit 0 +fi -[ "$(id -u)" -eq 0 ] || err "Run as root (needed to install the systemd service): sudo bash install-native.sh ..." [ -n "$SERVER" ] || err "--server is required (the central server's HTTPS address)" case "$SERVER" in https://*) ;; *) err "--server must be an https:// URL (the agent refuses cleartext)" ;; esac -command -v systemctl >/dev/null 2>&1 || err "systemd is required (systemctl not found)" command -v curl >/dev/null 2>&1 || err "curl is required" # Map machine architecture to the published self-contained runtime identifier. @@ -57,35 +285,37 @@ case "$(uname -m)" in *) err "Unsupported architecture: $(uname -m). Build from source (see the agent README)." ;; esac -echo "Installing Network Optimizer agent to ${INSTALL_DIR} (${RID})" +printf '\n%sNetwork Optimizer on-site agent%s\n' "$_b" "$_rst" +note "Installing to ${INSTALL_DIR} (${RID})" mkdir -p "$INSTALL_DIR" # Binaries are downloaded to a temp name and renamed into place: writing over a # binary while it is running fails with ETXTBSY, but rename swaps the directory # entry and any running process keeps its old inode until the restart below. +step "Downloading binaries" # Agent binary -echo "Downloading agent binary..." -curl -fSL "${RELEASE_BASE}/NetworkOptimizer.Agent-${RID}" -o "${INSTALL_DIR}/NetworkOptimizer.Agent.new" +curl -fsSL "${RELEASE_BASE}/NetworkOptimizer.Agent-${RID}" -o "${INSTALL_DIR}/NetworkOptimizer.Agent.new" chmod +x "${INSTALL_DIR}/NetworkOptimizer.Agent.new" mv -f "${INSTALL_DIR}/NetworkOptimizer.Agent.new" "${INSTALL_DIR}/NetworkOptimizer.Agent" +ok "agent (${RID})" # uwnspeedtest binary for site-local WAN speed tests; the agent resolves it next # to itself (AppContext.BaseDirectory/uwnspeedtest). -echo "Downloading WAN speed test binary..." -curl -fSL "${RELEASE_BASE}/uwnspeedtest-${RID}" -o "${INSTALL_DIR}/uwnspeedtest.new" +curl -fsSL "${RELEASE_BASE}/uwnspeedtest-${RID}" -o "${INSTALL_DIR}/uwnspeedtest.new" chmod +x "${INSTALL_DIR}/uwnspeedtest.new" mv -f "${INSTALL_DIR}/uwnspeedtest.new" "${INSTALL_DIR}/uwnspeedtest" +ok "WAN speed test helper" CONFIG="${INSTALL_DIR}/agent.json" +step "Configuring the agent" # Preserve an already-enrolled config so re-running the installer (e.g. to # update the binary) never wipes the persisted agent key. if grep -q '"agentKey"' "$CONFIG" 2>/dev/null; then - echo "Existing enrolled agent config found - keeping it." + note "Existing enrollment found - keeping agent.json" else [ -n "$TOKEN" ] || err "--token is required for a first-time install" - echo "Writing ${CONFIG}" { echo "{" echo " \"serverUrl\": \"${SERVER%/}\"," @@ -97,6 +327,7 @@ else fi printf '\n}\n' } > "$CONFIG" + ok "Wrote ${CONFIG}" fi # nginx serves the OpenSpeedTest page + the throughput-critical transfer legs @@ -109,13 +340,13 @@ fi # conf.d file and running `systemctl restart nginx` would hijack or bounce that # unrelated instance, which is unacceptable. We only borrow the nginx *binary*. if [ "$LAN_SPEED_TEST" = true ]; then - echo "Setting up a dedicated nginx instance for the LAN speed test..." + step "Setting up the LAN speed test (nginx)" if ! command -v nginx >/dev/null 2>&1; then if command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq nginx elif command -v dnf >/dev/null 2>&1; then dnf install -y -q nginx elif command -v yum >/dev/null 2>&1; then yum install -y -q nginx elif command -v apk >/dev/null 2>&1; then apk add --no-cache nginx - else echo "WARNING: could not install nginx automatically - install it and re-run to enable the LAN speed test."; fi + else warn "could not install nginx automatically - install it and re-run to enable the LAN speed test."; fi fi NGINX_BIN="$(command -v nginx 2>/dev/null || echo /usr/sbin/nginx)" @@ -125,7 +356,7 @@ if [ "$LAN_SPEED_TEST" = true ]; then WEBROOT="${INSTALL_DIR}/speedtest-web" RAW="https://raw.githubusercontent.com/Ozark-Connect/NetworkOptimizer/main" mkdir -p "$WEBROOT/assets/js" - echo "Downloading OpenSpeedTest..." + note "Fetching the OpenSpeedTest page" TARBALL="$(mktemp)"; TMPX="$(mktemp -d)" curl -fsSL "https://github.com/Ozark-Connect/NetworkOptimizer/archive/refs/heads/main.tar.gz" -o "$TARBALL" tar -xzf "$TARBALL" -C "$TMPX" --strip-components=3 "NetworkOptimizer-main/src/OpenSpeedTest" @@ -163,7 +394,7 @@ CFGJS -e 's/^\([[:space:]]*listen[[:space:]][^;]*\) ssl\([^;]*;\)/\1\2/' \ -e '/^[[:space:]]*ssl_/d' \ "${INSTALL_DIR}/nginx-speedtest-server.conf" - echo "AGENT_SPEEDTEST_TLS=0 - LAN speed test will serve plain http on port 3000." + note "AGENT_SPEEDTEST_TLS=0 - serving plain http on port 3000" else # Persisted self-signed cert for the LAN speed test's TLS listener (secure context # for the browser Geolocation API / GPS-tagged results, no per-site reverse proxy). @@ -180,7 +411,7 @@ CFGJS -keyout "$CERTDIR/key.pem" -out "$CERTDIR/cert.pem" \ -subj "/CN=${CN:-agent}" -addext "subjectAltName=$SAN" >/dev/null 2>&1 \ && chmod 600 "$CERTDIR/key.pem" \ - || echo "WARNING: self-signed cert generation failed - the LAN speed test won't serve over TLS." + || warn "self-signed cert generation failed - the LAN speed test won't serve over TLS." fi sed -i \ -e "s#__CERTFILE__#${CERTDIR}/cert.pem#" \ @@ -195,6 +426,15 @@ CFGJS -e "s#__SERVERCONF__#${INSTALL_DIR}/nginx-speedtest-server.conf#" \ "${INSTALL_DIR}/nginx-speedtest.conf" + # If the install dir isn't world-traversable (e.g. under /root, mode 700), + # unprivileged nginx workers can't reach the webroot and every request 403s. + # Run the workers as root in that case so the page serves wherever it lives; + # a world-traversable dir (the /opt default) keeps the safer unprivileged user. + if _webroot_needs_root_worker "$WEBROOT"; then + sed -i "1i user root;" "${INSTALL_DIR}/nginx-speedtest.conf" + note "${INSTALL_DIR} isn't world-traversable, so the speed test nginx runs its workers as root" + fi + # Dedicated systemd unit for OUR nginx master - separate from the system one. cat > /etc/systemd/system/netopt-speedtest-nginx.service </dev/null 2>&1; then + maybe_fix_apparmor_nginx + fi + if "$NGINX_BIN" -t -c "${INSTALL_DIR}/nginx-speedtest.conf" >/dev/null 2>&1; then systemctl daemon-reload # Enable now, but START it below, after the agent unit is installed and # running - nginx BindsTo the agent, so starting it before the agent exists # would immediately stop it again. - systemctl enable netopt-speedtest-nginx.service + systemctl enable --quiet netopt-speedtest-nginx.service START_SPEEDTEST_NGINX=1 - echo "Dedicated nginx for OpenSpeedTest on port 3000 will start with the agent (netopt-speedtest-nginx.service)." + ok "LAN speed test ready on port 3000 (starts with the agent)" else - echo "WARNING: nginx config test failed - the LAN speed test page won't serve." - echo "Diagnose with: $NGINX_BIN -t -c ${INSTALL_DIR}/nginx-speedtest.conf" + warn "nginx config test failed - the LAN speed test page won't serve." + print_speedtest_apparmor_hint + note "Diagnose: sudo $NGINX_BIN -t -c ${INSTALL_DIR}/nginx-speedtest.conf" fi fi fi # systemd unit -echo "Installing ${SERVICE_NAME}.service" +step "Installing the agent service" cat > "/etc/systemd/system/${SERVICE_NAME}.service" <