fix for tests

Author: KSemenenko

AGENTS.md

@@ -115,6 +115,11 @@ If no new rule is detected -> do not update the file.
 - Always run required builds and tests yourself; do not ask the user to execute them (explicit user directive)
 - Commit messages are short and imperative; common prefixes in history include `fix:`, `tests:`, and `code style`
 
+### Reviews
+
+- For code reviews, be extra thorough and explicitly call out low-value/AI-sounding changes and whether changes actually improve behavior, performance, or safety
+- Never run SignalR work on Orleans scheduler/context; offload to dedicated tasks/threads to avoid blocking
+
 ### Documentation (ALL TASKS)
 
 - All docs live in `docs/` (or `.wiki/`)
@@ -143,6 +148,7 @@ If no new rule is detected -> do not update the file.
 ### Testing (ALL TASKS)
 
 - Framework: xUnit + Shouldly; tests live in `ManagedCode.Orleans.SignalR.Tests` and end with `*Tests.cs`
+- Avoid introducing new `[GenerateSerializer]` state types in tests; reuse production models to keep Orleans serialization types in product code.
 - Integration tests use Orleans TestingHost with fixtures in `ManagedCode.Orleans.SignalR.Tests/Cluster` and the minimal host in `ManagedCode.Orleans.SignalR.Tests/TestApp`
 - Prefer TDD for new behaviour and bugfixes: write a failing test first, then implement the smallest change to make it pass, then refactor safely
 - Every behaviour change needs sufficient automated tests to cover its cases; one is the minimum, not the target
@@ -180,6 +186,10 @@ If no new rule is detected -> do not update the file.
 - Never use `ConfigureAwait(false)`
 - No magic literals - extract to constants, enums, config
 
+### Comments
+
+- When offloading SignalR observer sends with Task.Run, add a critical comment explaining it must not run on the Orleans scheduler to avoid blocking
+
 ### Critical (NEVER violate)
 
 - Never commit secrets, keys, connection strings

ManagedCode.Orleans.SignalR.Core/Config/OrleansSignalROptions.cs

@@ -63,7 +63,8 @@ public class OrleansSignalROptions
     public int MaxQueuedMessagesPerUser { get; set; } = 100;
 
     /// <summary>
-    ///     Number of consecutive failures before an observer is considered dead and removed.
+    ///     Number of consecutive failures before the circuit breaker opens.
+    ///     When the circuit breaker is disabled or the grace period is zero, the observer is removed.
     ///     Set to 0 to disable failure tracking.
     ///     The default value is 3.
     /// </summary>
@@ -111,45 +112,4 @@ public class OrleansSignalROptions
     /// </summary>
     public int MaxBufferedMessagesPerObserver { get; set; } = 50;
 
-    /// <summary>
-    ///     Maximum number of connections allowed per partition grain.
-    ///     New connections are rejected when the limit is exceeded.
-    ///     Set to 0 to disable connection limits (not recommended for production).
-    ///     The default value is 100,000.
-    /// </summary>
-    public int MaxConnectionsPerPartition { get; set; } = 100_000;
-
-    /// <summary>
-    ///     Maximum number of groups per partition grain.
-    ///     New groups are rejected when the limit is exceeded.
-    ///     Set to 0 to disable group limits.
-    ///     The default value is 50,000.
-    /// </summary>
-    public int MaxGroupsPerPartition { get; set; } = 50_000;
-
-    /// <summary>
-    ///     Timeout for slow client message delivery.
-    ///     Connections that cannot receive messages within this time may be terminated.
-    ///     The default value is 10 seconds.
-    /// </summary>
-    public TimeSpan SlowClientTimeout { get; set; } = TimeSpan.FromSeconds(10);
-
-    /// <summary>
-    ///     Enables backpressure handling for slow clients.
-    ///     When enabled, messages to slow clients are dropped or the connection is terminated.
-    ///     The default value is true.
-    /// </summary>
-    public bool EnableSlowClientHandling { get; set; } = true;
-
-    /// <summary>
-    ///     Maximum number of pending messages allowed per connection before backpressure is applied.
-    ///     The default value is 1000.
-    /// </summary>
-    public int MaxPendingMessagesPerConnection { get; set; } = 1000;
-
-    /// <summary>
-    ///     Enables metrics collection for monitoring and diagnostics.
-    ///     The default value is true.
-    /// </summary>
-    public bool EnableMetrics { get; set; } = true;
 }

ManagedCode.Orleans.SignalR.Core/Helpers/RetryHelper.cs

@@ -8,7 +8,8 @@ namespace ManagedCode.Orleans.SignalR.Core.SignalR.Observers;
 
 /// <summary>
 /// Tracks observer health by monitoring delivery failures with circuit breaker support.
-/// Observers exceeding the failure threshold have their circuit opened to prevent cascade failures.
+/// When circuit breaker is enabled, failures open the circuit and optionally buffer messages
+/// during a grace period before observers are removed.
 /// Supports graceful expiration with message buffering for timing edge cases.
 ///
 /// Note: This class is designed to be used within Orleans grains which provide single-threaded
@@ -82,7 +83,8 @@ public FailureResult RecordFailure(string connectionId, Exception? exception = n
                 CircuitBreakerEnabled,
                 _failureThreshold,
                 _circuitOpenDuration,
-                _halfOpenTestInterval);
+                _halfOpenTestInterval,
+                markDeadOnCircuitOpen: !_gracePeriodBuffer.IsEnabled);
             _healthStates[connectionId] = state;
         }
 
@@ -311,6 +313,7 @@ private sealed class ObserverHealthState
         private readonly TimeSpan _failureWindow;
         private readonly bool _circuitBreakerEnabled;
         private readonly int _failureThreshold;
+        private readonly bool _markDeadOnCircuitOpen;
         private readonly List<long> _failureTimestamps = new();
         private readonly ObserverCircuitBreaker? _circuitBreaker;
 
@@ -319,11 +322,13 @@ public ObserverHealthState(
             bool circuitBreakerEnabled,
             int failureThreshold,
             TimeSpan circuitOpenDuration,
-            TimeSpan halfOpenTestInterval)
+            TimeSpan halfOpenTestInterval,
+            bool markDeadOnCircuitOpen)
         {
             _failureWindow = failureWindow;
             _circuitBreakerEnabled = circuitBreakerEnabled;
             _failureThreshold = failureThreshold;
+            _markDeadOnCircuitOpen = markDeadOnCircuitOpen;
 
             if (circuitBreakerEnabled)
             {
@@ -373,17 +378,28 @@ public FailureResult RecordFailure(Exception? exception)
             LastException = exception;
 
             var failureCount = _failureTimestamps.Count;
-            var circuitOpened = _circuitBreaker?.RecordFailure(exception) ?? false;
 
-            if (failureCount >= _failureThreshold)
+            if (_circuitBreaker is not null)
             {
-                IsDead = true;
-                return FailureResult.Dead;
+                var circuitOpened = _circuitBreaker.RecordFailure(exception);
+                if (circuitOpened)
+                {
+                    if (_markDeadOnCircuitOpen)
+                    {
+                        IsDead = true;
+                        return FailureResult.Dead;
+                    }
+
+                    return FailureResult.CircuitOpened;
+                }
+
+                return FailureResult.Healthy;
             }
 
-            if (circuitOpened)
+            if (failureCount >= _failureThreshold)
             {
-                return FailureResult.CircuitOpened;
+                IsDead = true;
+                return FailureResult.Dead;
             }
 
             return FailureResult.Healthy;