Ozark-Connect
diff --git a/‎NetworkOptimizer.sln‎
Lines changed: 15 additions & 0 deletions b/‎NetworkOptimizer.sln‎
Lines changed: 15 additions & 0 deletions
diff --git a/‎src/NetworkOptimizer.Agent/ProbeRunner.cs‎
Lines changed: 15 additions & 12 deletions b/‎src/NetworkOptimizer.Agent/ProbeRunner.cs‎
Lines changed: 15 additions & 12 deletions
diff --git a/‎src/NetworkOptimizer.Agent/Program.cs‎
Lines changed: 45 additions & 10 deletions b/‎src/NetworkOptimizer.Agent/Program.cs‎
Lines changed: 45 additions & 10 deletions
diff --git a/‎src/NetworkOptimizer.Agent/SnmpRunner.cs‎
Lines changed: 15 additions & 12 deletions b/‎src/NetworkOptimizer.Agent/SnmpRunner.cs‎
Lines changed: 15 additions & 12 deletions
@@ -59,6 +59,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetworkOptimizer.Agent", "s
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetworkOptimizer.AgentProtocol", "src\NetworkOptimizer.AgentProtocol\NetworkOptimizer.AgentProtocol.csproj", "{CBF381E5-F3CF-4A4F-85C8-DD5BFAB51741}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NetworkOptimizer.AgentProtocol.Tests", "tests\NetworkOptimizer.AgentProtocol.Tests\NetworkOptimizer.AgentProtocol.Tests.csproj", "{EC6241C2-8CD8-4321-B93C-3ACC408050E9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -381,6 +383,18 @@ Global
 		{CBF381E5-F3CF-4A4F-85C8-DD5BFAB51741}.Release|x64.Build.0 = Release|Any CPU
 		{CBF381E5-F3CF-4A4F-85C8-DD5BFAB51741}.Release|x86.ActiveCfg = Release|Any CPU
 		{CBF381E5-F3CF-4A4F-85C8-DD5BFAB51741}.Release|x86.Build.0 = Release|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Debug|x64.Build.0 = Debug|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Debug|x86.Build.0 = Debug|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Release|x64.ActiveCfg = Release|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Release|x64.Build.0 = Release|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Release|x86.ActiveCfg = Release|Any CPU
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -401,6 +415,7 @@ Global
 		{EC72C9AD-625C-4AA8-A7CC-744515E06F1E} = {5691A6DD-53B9-4CE0-A3C9-3D4F815E2120}
 		{01F2AE32-FE55-4892-B91B-CE2BC964F29D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{CBF381E5-F3CF-4A4F-85C8-DD5BFAB51741} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{EC6241C2-8CD8-4321-B93C-3ACC408050E9} = {5691A6DD-53B9-4CE0-A3C9-3D4F815E2120}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F7E6D5C4-B3A2-9180-7F6E-5D4C3B2A1908}
 
@@ -10,13 +10,16 @@ namespace NetworkOptimizer.Agent;
 /// The site's latency/loss probe engine. The server pushes the site's
 /// monitoring targets over the tunnel; this runner probes each on its own
 /// cadence (2 s scheduler tick, per-target intervals, bounded concurrency,
-/// mirroring the server's latency tier) and streams every result straight
-/// back. Created per tunnel connection: the server re-pushes config on every
-/// connect, and results have nowhere to go while the tunnel is down.
+/// mirroring the server's latency tier) and enqueues every result into the
+/// store-and-forward <see cref="ResultBuffer"/>. Runs for the life of the
+/// agent process: probing continues through tunnel outages - which is exactly
+/// when latency/loss data matters most - with the buffer holding the backlog
+/// and the last pushed target set staying in effect until the server
+/// re-pushes on reconnect.
 /// </summary>
 public sealed class ProbeRunner
 {
-    private readonly Func<AgentMessage, bool> _send;
+    private readonly Action<AgentMessage> _send;
     private readonly string? _defaultSourceIp;
     private readonly LocalProbeExecutor _executor = new(NullLogger<LocalProbeExecutor>.Instance);
     private readonly ConcurrentDictionary<string, DateTime> _lastProbed = new();
@@ -27,7 +30,7 @@ public sealed class ProbeRunner
     /// own. This is the probe-only multi-WAN deployment knob: the gateway
     /// policy-routes this source IP out a specific WAN.
     /// </param>
-    public ProbeRunner(Func<AgentMessage, bool> send, string? defaultSourceIp = null)
+    public ProbeRunner(Action<AgentMessage> send, string? defaultSourceIp = null)
     {
         _send = send;
         _defaultSourceIp = string.IsNullOrEmpty(defaultSourceIp) ? null : defaultSourceIp;
@@ -42,13 +45,13 @@ public void UpdateConfig(ProbeConfig config)
     public async Task RunAsync(CancellationToken ct)
     {
         // Startup grace, mirroring the server latency tier's 20s initial delay
-        // (MonitoringCollectionAgent.RunTierAsync). A fresh ProbeRunner starts on every
-        // tunnel connect - the agent (re)starting, OR a NO-server restart that dropped
-        // and re-established the tunnel - and every target is "due" immediately. Probing
-        // the instant the process/link comes up catches the network still settling
-        // (routes, source-IP binds, the target device itself rebooting alongside) and
-        // records a false loss/latency spike right at the restart boundary. Wait for it
-        // to settle before the first tick, so no boundary sample is taken.
+        // (MonitoringCollectionAgent.RunTierAsync). At agent process start every target
+        // is "due" immediately, and probing the instant the process comes up catches
+        // the network still settling (routes, source-IP binds, the target device itself
+        // rebooting alongside) and records a false loss/latency spike right at the
+        // start boundary. Wait for it to settle before the first tick, so no boundary
+        // sample is taken. Tunnel reconnects no longer restart this runner, so no
+        // grace (or gap) applies there - probing runs continuously across outages.
         try { await Task.Delay(TimeSpan.FromSeconds(20), ct); }
         catch (OperationCanceledException) { return; }
 
 
@@ -3,6 +3,7 @@
 using System.Runtime.InteropServices;
 using System.Text.Json;
 using NetworkOptimizer.Agent;
+using NetworkOptimizer.AgentProtocol;
 
 // On-site agent skeleton: enrolls against the central Network Optimizer server
 // with a one-time token, persists its agent key and site slug, then heartbeats.
@@ -185,26 +186,41 @@ static bool IsHttpsUrl(string url) =>
         speedTestServer is { } relayServer ? relayServer.PostIperf3ResultAsync : null, cts.Token);
 }
 
+// Monitoring collectors and their store-and-forward buffer live for the whole
+// process, not per tunnel connection: probing and SNMP polling continue on the
+// last pushed config while the tunnel is down - exactly when outage data
+// matters most - and the buffered backlog (12 h / 64 MB caps, oldest dropped
+// first) replays over the tunnel on reconnect. Only started when a tunnel is
+// configured; without one no config ever arrives and there is nothing to run.
+ResultBuffer? resultBuffer = null;
+ProbeRunner? probeRunner = null;
+SnmpRunner? snmpRunner = null;
+Task? probeTask = null;
+Task? snmpTask = null;
+if (!string.IsNullOrEmpty(config.TunnelUrl))
+{
+    resultBuffer = new ResultBuffer();
+    probeRunner = new ProbeRunner(resultBuffer.Enqueue, config.ProbeSourceIp);
+    snmpRunner = new SnmpRunner(resultBuffer.Enqueue);
+    probeTask = probeRunner.RunAsync(cts.Token);
+    snmpTask = snmpRunner.RunAsync(cts.Token);
+}
+
 // Prefer the persistent gRPC tunnel; REST heartbeats keep the agent visible as
 // Online whenever the tunnel is unavailable (tunnel disabled server-side, an
 // older server, or a network drop between reconnect attempts).
 while (!cts.IsCancellationRequested)
 {
     if (!string.IsNullOrEmpty(config.TunnelUrl))
     {
-        // The probe runner lives and dies with its tunnel connection: the
-        // server re-pushes probe config on every connect, and results have
-        // nowhere to go while the tunnel is down.
         using var connectionCts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token);
         var tunnel = new TunnelClient();
-        var probeRunner = new ProbeRunner(tunnel.TrySend, config.ProbeSourceIp);
-        var snmpRunner = new SnmpRunner(tunnel);
         var proxyHandler = new ProxyHandler(tunnel, proxyPolicy);
         var iperf3ClientRunner = new Iperf3ClientRunner(tunnel);
         var uwnClientRunner = new UwnClientRunner(tunnel);
         var probeRequestRunner = new ProbeRequestRunner(tunnel);
-        tunnel.OnProbeConfig = probeRunner.UpdateConfig;
-        tunnel.OnSnmpConfig = snmpRunner.UpdateConfig;
+        tunnel.OnProbeConfig = probeRunner!.UpdateConfig;
+        tunnel.OnSnmpConfig = snmpRunner!.UpdateConfig;
         tunnel.OnWanSpeedTestConfig = wanConfig => speedTestServer?.UpdateWanServers(
             wanConfig.Servers.Select(s => new SpeedTestServer.WanServerEntry(s.ServerId, s.Url)).ToList(),
             wanConfig.DefaultServerId);
@@ -215,8 +231,17 @@ static bool IsHttpsUrl(string url) =>
         tunnel.OnIperf3Request = iperf3ClientRunner.HandleAsync;
         tunnel.OnUwnRequest = uwnClientRunner.HandleAsync;
         tunnel.OnProbeRequest = probeRequestRunner.HandleAsync;
-        var probeTask = probeRunner.RunAsync(connectionCts.Token);
-        var snmpTask = snmpRunner.RunAsync(connectionCts.Token);
+        // Server confirms persisted result frames; trim them from the buffer so
+        // only unacked data is retained for replay.
+        tunnel.OnResultAck = seq => resultBuffer!.MarkAcked(seq);
+        var backlog = resultBuffer!.Count;
+        if (backlog > 0)
+            Console.WriteLine($"Buffered backlog: {backlog} result message(s) (~{resultBuffer.ApproxBytes / 1024} KB) will flush once the tunnel connects");
+        var droppedOffline = resultBuffer.TakeDroppedCount();
+        if (droppedOffline > 0)
+            Console.Error.WriteLine($"Result buffer caps dropped the {droppedOffline} oldest message(s) while the tunnel was down");
+
+        var drainTask = tunnel.DrainResultsAsync(resultBuffer, connectionCts.Token);
         try
         {
             await tunnel.RunAsync(config.TunnelUrl, config.AgentKey!, version, lanIp, config.IgnoreSslErrors, cts.Token);
@@ -233,7 +258,9 @@ static bool IsHttpsUrl(string url) =>
         finally
         {
             connectionCts.Cancel();
-            try { await Task.WhenAll(probeTask, snmpTask); } catch (OperationCanceledException) { }
+            // Nothing to salvage: the drain only peeks, so every unacked frame is
+            // still in the buffer and replays on the next connection.
+            try { await drainTask; } catch (OperationCanceledException) { }
         }
     }
 
@@ -270,6 +297,14 @@ static bool IsHttpsUrl(string url) =>
 {
     try { await iperf3Task; } catch (OperationCanceledException) { }
 }
+if (probeTask != null)
+{
+    try { await probeTask; } catch (OperationCanceledException) { }
+}
+if (snmpTask != null)
+{
+    try { await snmpTask; } catch (OperationCanceledException) { }
+}
 
 Console.WriteLine("Agent stopped");
 return 0;
 
@@ -8,14 +8,16 @@ namespace NetworkOptimizer.Agent;
 /// <summary>
 /// The site's SNMP poller. The server pushes the site's SNMP credentials and
 /// device list over the tunnel; this runner polls interface counters (fast
-/// cadence) and device health (medium cadence) locally and streams the raw
-/// samples back. Rate computation and storage happen server-side, mirroring
-/// the central collection agent's split. Lives and dies with its tunnel
-/// connection, like the probe runner.
+/// cadence) and device health (medium cadence) locally and enqueues the raw
+/// samples into the store-and-forward <see cref="ResultBuffer"/>. Rate
+/// computation and storage happen server-side, mirroring the central
+/// collection agent's split. Runs for the life of the agent process, like the
+/// probe runner: polling continues through tunnel outages on the last pushed
+/// config, and the buffered backlog replays on reconnect.
 /// </summary>
 public sealed class SnmpRunner
 {
-    private readonly TunnelClient _tunnel;
+    private readonly Action<AgentMessage> _send;
     private volatile SnmpConfig? _config;
     private SnmpPoller? _poller;
     private string _pollerKey = "";
@@ -25,9 +27,9 @@ public sealed class SnmpRunner
     // the agent either. Keyed by device MAC.
     private readonly SnmpFailureTracker _failures = new();
 
-    public SnmpRunner(TunnelClient tunnel)
+    public SnmpRunner(Action<AgentMessage> send)
     {
-        _tunnel = tunnel;
+        _send = send;
     }
 
     public void UpdateConfig(SnmpConfig config)
@@ -43,7 +45,9 @@ public Task RunAsync(CancellationToken ct) =>
 
     /// <summary>
     /// Handles the server's on-demand "Test OID" request: GET the single OID once against a
-    /// site-local device and return the raw value (correlated by request id).
+    /// site-local device and return the raw value (correlated by request id). The reply rides
+    /// the result buffer like everything else; if the tunnel drops first, the server has
+    /// already timed the request out and ignores the stale request id on replay.
     /// </summary>
     public async Task HandleOidQueryAsync(SnmpOidQuery query, CancellationToken ct)
     {
@@ -74,8 +78,7 @@ public async Task HandleOidQueryAsync(SnmpOidQuery query, CancellationToken ct)
         }
 
         Console.WriteLine($"OID test request {query.RequestId}: success={result.Success} value={result.Value} error={result.Error}");
-        try { await _tunnel.SendAsync(new AgentMessage { SnmpOidResult = result }, ct); }
-        catch (Exception ex) { Console.Error.WriteLine($"Failed to send OID test result {query.RequestId}: {ex.Message}"); }
+        _send(new AgentMessage { SnmpOidResult = result });
     }
 
     /// <summary>Interface counters on the fast cadence.</summary>
@@ -131,7 +134,7 @@ await ForEachDeviceAsync(config, async device =>
                         });
                     }
                     if (batch.Interfaces.Count > 0)
-                        await _tunnel.SendAsync(new AgentMessage { SnmpResults = batch }, ct);
+                        _send(new AgentMessage { SnmpResults = batch });
                 }, ct);
             }
 
@@ -181,7 +184,7 @@ await ForEachDeviceAsync(config, async device =>
                     await PollCustomOidsAsync(poller, ip, device, config, batch);
 
                     if (batch.Health.Count > 0 || batch.CustomOids.Count > 0)
-                        await _tunnel.SendAsync(new AgentMessage { SnmpResults = batch }, ct);
+                        _send(new AgentMessage { SnmpResults = batch });
                 }, ct);
             }