feat: add otel metrics

ascott18 · ascott18 · commit 7c352465311b · 2025-10-02T14:54:39.000-07:00
diff --git a/README.md b/README.md
@@ -78,16 +78,20 @@ services.Configure<SqlServerOptions>(Configuration.GetSection("SignalR:SqlServer
 ## OpenTelemetry Support
 
 This library includes OpenTelemetry instrumentation that wraps background database queries in activities, making them more easily identified and grouped in your collected telemetry.
-
+a
 ### Setup
 
-To enable OpenTelemetry collection of these trace spans, add the source to your tracing configuration:
+To enable OpenTelemetry collection of these trace spans and metrics, add the source and meter to your configuration:
 
 ``` cs
 builder.Services.AddOpenTelemetry()
     .WithTracing(tracing => tracing
         .AddSource("IntelliTect.AspNetCore.SignalR.SqlServer")
         // ... other instrumentation
+    )
+    .WithMetrics(metrics => metrics
+        .AddMeter("IntelliTect.AspNetCore.SignalR.SqlServer")
+        // ... other instrumentation
     );
 ```
 
@@ -100,6 +104,17 @@ The library creates activities for the following operations:
 - `SignalR.SqlServer.Poll` - Polling operations (database reads, when Service Broker is not used)
 - `SignalR.SqlServer.Publish` - Message publishing operations (database writes)
 
+### Metrics
+
+The library also provides metrics to help monitor the health and performance of the SQL Server backplane:
+
+- `signalr.sqlserver.poll_delay` - Histogram showing the distribution of polling delay intervals, useful for understanding backoff patterns and system health
+- `signalr.sqlserver.query_duration` - Histogram tracking the duration of SQL Server query execution for reading messages
+- `signalr.sqlserver.rows_read_total` - Counter tracking the total number of message rows read from SQL Server
+- `signalr.sqlserver.rows_written_total` - Counter tracking the total number of message rows written to SQL Server
+
+These metrics help you understand polling patterns, database performance, message throughput, and can be useful for tuning performance or identifying when Service Broker fallback to polling occurs.
+
 ### Filtering Noise
 
 Since the SQL Server backplane performs frequent polling operations, you may want to filter out successful, fast queries to reduce trace noise. 
diff --git a/demo/DemoServer/Program.cs b/demo/DemoServer/Program.cs
@@ -34,6 +34,7 @@
     .WithMetrics(metrics => metrics
         .AddAspNetCoreInstrumentation()
         .AddSqlClientInstrumentation()
+        .AddMeter("IntelliTect.AspNetCore.SignalR.SqlServer")
     )
     .WithTracing(tracing => tracing
         .AddSource(builder.Environment.ApplicationName)
diff --git a/src/IntelliTect.AspNetCore.SignalR.SqlServer/Internal/SqlServer/SqlReceiver.cs b/src/IntelliTect.AspNetCore.SignalR.SqlServer/Internal/SqlServer/SqlReceiver.cs
@@ -6,6 +6,7 @@
 using System.Data;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.Metrics;
 using System.Globalization;
 using System.Threading;
 using System.Threading.Tasks;
@@ -43,6 +44,21 @@ internal class SqlReceiver : IDisposable
         private readonly string _maxIdSql = "SELECT [PayloadId] FROM [{0}].[{1}_Id]";
         private readonly string _selectSql = "SELECT [PayloadId], [Payload], [InsertedOn] FROM [{0}].[{1}] WHERE [PayloadId] > @PayloadId";
         private readonly TimeSpan _activityMaxDuration = TimeSpan.FromMinutes(10);
+        
+        private static readonly Histogram<double> _pollDelayHistogram = SqlServerOptions.Meter.CreateHistogram<double>(
+            "signalr.sqlserver.poll_delay",
+            "ms",
+            "Distribution of polling delay intervals showing backoff patterns");
+            
+        private static readonly Counter<long> _rowsReadCounter = SqlServerOptions.Meter.CreateCounter<long>(
+            "signalr.sqlserver.rows_read_total",
+            "rows",
+            "Total number of message rows read from SQL Server");
+            
+        private static readonly Histogram<double> _queryDurationHistogram = SqlServerOptions.Meter.CreateHistogram<double>(
+            "signalr.sqlserver.query_duration",
+            "ms",
+            "Duration of SQL Server query execution for reading messages");
             
 
         public SqlReceiver(SqlServerOptions options, ILogger logger, string tableName, string tracePrefix)
@@ -222,7 +238,6 @@ private async Task PollingLoop(CancellationToken cancellationToken)
 
             try
             {
-
                 var delays = _updateLoopRetryDelays;
                 for (var retrySetIndex = 0; retrySetIndex < delays.Length; retrySetIndex++)
                 {
@@ -253,6 +268,10 @@ private async Task PollingLoop(CancellationToken cancellationToken)
                             }
 
                             recordCount = await ReadRows(null);
+                            
+                            // Record polling delay distribution to show backoff patterns
+                            _pollDelayHistogram.Record(retryDelay, 
+                                new KeyValuePair<string, object?>("signalr.hub", _tracePrefix));
                         }
                         catch (TaskCanceledException) { return; }
                         catch (Exception ex)
@@ -336,6 +355,7 @@ private async Task<int> ReadRows(Action<SqlCommand>? beforeExecute)
             beforeExecute?.Invoke(command);
 
             // Fetch any rows that might already be available after the last PayloadId.
+            var start = DateTime.UtcNow;
             var reader = await command.ExecuteReaderAsync();
             var recordCount = 0;
 
@@ -345,6 +365,18 @@ private async Task<int> ReadRows(Action<SqlCommand>? beforeExecute)
                 await ProcessRecord(reader);
             }
 
+            // Record query duration and rows read metrics
+            if (_queryDurationHistogram.Enabled)
+            {
+                _queryDurationHistogram.Record((DateTime.UtcNow - start).TotalMilliseconds, 
+                    new KeyValuePair<string, object?>("signalr.hub", _tracePrefix));
+            }
+                
+            if (recordCount > 0 && _rowsReadCounter.Enabled)
+            {
+                _rowsReadCounter.Add(recordCount, new KeyValuePair<string, object?>("signalr.hub", _tracePrefix));
+            }
+
             return recordCount;
         }
 
diff --git a/src/IntelliTect.AspNetCore.SignalR.SqlServer/Internal/SqlServer/SqlSender.cs b/src/IntelliTect.AspNetCore.SignalR.SqlServer/Internal/SqlServer/SqlSender.cs
@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Metrics;
 using System.Globalization;
 using System.Threading.Tasks;
 using Microsoft.Data.SqlClient;
@@ -12,15 +13,22 @@ namespace IntelliTect.AspNetCore.SignalR.SqlServer.Internal
 {
     internal class SqlSender
     {
+        private static readonly Counter<long> _rowsWrittenCounter = SqlServerOptions.Meter.CreateCounter<long>(
+            "signalr.sqlserver.rows_written_total",
+            "rows",
+            "Total number of message rows written to SQL Server");
+            
         private readonly string _insertDml;
         private readonly ILogger _logger;
         private readonly SqlServerOptions _options;
+        private readonly string _hubName;
 
-        public SqlSender(SqlServerOptions options, ILogger logger, string tableName)
+        public SqlSender(SqlServerOptions options, ILogger logger, string tableName, string hubName)
         {
             _options = options;
             _insertDml = BuildInsertString(tableName);
             _logger = logger;
+            _hubName = hubName;
         }
 
         private string BuildInsertString(string tableName)
@@ -43,6 +51,9 @@ public async Task Send(byte[] message)
             await connection.OpenAsync();
 
             await command.ExecuteNonQueryAsync();
+            
+            // Record rows written metric
+            _rowsWrittenCounter.Add(1, new KeyValuePair<string, object?>("signalr.hub", _hubName));
         }
     }
 }
diff --git a/src/IntelliTect.AspNetCore.SignalR.SqlServer/Internal/SqlServer/SqlStream.cs b/src/IntelliTect.AspNetCore.SignalR.SqlServer/Internal/SqlServer/SqlStream.cs
@@ -26,7 +26,7 @@ public SqlStream(SqlServerOptions options, ILogger logger, int streamIndex, stri
             _logger = logger;
             _tracePrefix = tracePrefix;
 
-            _sender = new SqlSender(options, logger, tableName);
+            _sender = new SqlSender(options, logger, tableName, _tracePrefix);
             _receiver = new SqlReceiver(options, logger, tableName, _tracePrefix);
         }
 
diff --git a/src/IntelliTect.AspNetCore.SignalR.SqlServer/SqlServerOptions.cs b/src/IntelliTect.AspNetCore.SignalR.SqlServer/SqlServerOptions.cs
@@ -4,6 +4,7 @@
 using System;
 using System.ComponentModel.DataAnnotations;
 using System.Diagnostics;
+using System.Diagnostics.Metrics;
 using System.IO;
 using System.Net;
 using System.Threading.Tasks;
@@ -16,6 +17,7 @@ namespace IntelliTect.AspNetCore.SignalR.SqlServer
     public class SqlServerOptions
     {
         internal static readonly ActivitySource ActivitySource = new("IntelliTect.AspNetCore.SignalR.SqlServer");
+        internal static readonly Meter Meter = new("IntelliTect.AspNetCore.SignalR.SqlServer");
         
         /// <summary>
         /// Shared lock to prevent multiple concurrent installs against the same DB.
diff --git a/test/IntelliTect.AspNetCore.SignalR.SqlServer.Tests/SqlServerEndToEndTests.cs b/test/IntelliTect.AspNetCore.SignalR.SqlServer.Tests/SqlServerEndToEndTests.cs
@@ -64,7 +64,7 @@ public async Task CanSendAndReceivePayloads_WithServiceBroker_UnderHeavyLoad()
 
             var prefix = nameof(CanSendAndReceivePayloads_WithServiceBroker_UnderHeavyLoad);
             var installer = new SqlInstaller(options, NullLogger.Instance, prefix, prefix);
-            var receiver = new SqlReceiver(options, NullLogger.Instance, prefix + "_0", "");
+            var receiver = new SqlReceiver(options, NullLogger.Instance, prefix + "_0", "TestHub");
 
             var receivedMessages = new ConcurrentBag<byte[]>();
             await installer.Install();
@@ -83,7 +83,7 @@ public async Task CanSendAndReceivePayloads_WithServiceBroker_UnderHeavyLoad()
             // the hub to be sending messages.
             int numSenders = 100;
             int numSent = 0;
-            var sender = new SqlSender(options, NullLogger.Instance, prefix + "_0");
+            var sender = new SqlSender(options, NullLogger.Instance, prefix + "_0", "TestHub");
             for (int i = 0; i < numSenders; i++)
             {
                 _ = Task.Run(async () =>
@@ -116,8 +116,8 @@ public async Task CanSendAndReceivePayloads_WithServiceBroker_UnderHeavyLoad()
         private async Task RunCore(SqlServerOptions options, string prefix)
         {
             var installer = new SqlInstaller(options, NullLogger.Instance, prefix, prefix);
-            var sender = new SqlSender(options, NullLogger.Instance, prefix + "_0");
-            var receiver = new SqlReceiver(options, NullLogger.Instance, prefix + "_0", "");
+            var sender = new SqlSender(options, NullLogger.Instance, prefix + "_0", "TestHub");
+            var receiver = new SqlReceiver(options, NullLogger.Instance, prefix + "_0", "TestHub");
 
             var receivedMessages = new ConcurrentBag<byte[]>();
             var receivedEvent = new SemaphoreSlim(0);

Original file line number	Diff line number	Diff line change
`@@ -34,6 +34,7 @@`
`34`	`34`	`.WithMetrics(metrics => metrics`
`35`	`35`	`.AddAspNetCoreInstrumentation()`
`36`	`36`	`.AddSqlClientInstrumentation()`
	`37`	`+ .AddMeter("IntelliTect.AspNetCore.SignalR.SqlServer")`
`37`	`38`	`)`
`38`	`39`	`.WithTracing(tracing => tracing`
`39`	`40`	`.AddSource(builder.Environment.ApplicationName)`
Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,7 @@`
`3`	`3`
`4`	`4`	`using System;`
`5`	`5`	`using System.Collections.Generic;`
	`6`	`+using System.Diagnostics.Metrics;`
`6`	`7`	`using System.Globalization;`
`7`	`8`	`using System.Threading.Tasks;`
`8`	`9`	`using Microsoft.Data.SqlClient;`
`@@ -12,15 +13,22 @@ namespace IntelliTect.AspNetCore.SignalR.SqlServer.Internal`
`12`	`13`	`{`
`13`	`14`	`internal class SqlSender`
`14`	`15`	`{`
	`16`	`+ private static readonly Counter<long> _rowsWrittenCounter = SqlServerOptions.Meter.CreateCounter<long>(`
	`17`	`+ "signalr.sqlserver.rows_written_total",`
	`18`	`+ "rows",`
	`19`	`+ "Total number of message rows written to SQL Server");`
	`20`	`+`
`15`	`21`	`private readonly string _insertDml;`
`16`	`22`	`private readonly ILogger _logger;`
`17`	`23`	`private readonly SqlServerOptions _options;`
	`24`	`+ private readonly string _hubName;`
`18`	`25`
`19`		`- public SqlSender(SqlServerOptions options, ILogger logger, string tableName)`
	`26`	`+ public SqlSender(SqlServerOptions options, ILogger logger, string tableName, string hubName)`
`20`	`27`	`{`
`21`	`28`	`_options = options;`
`22`	`29`	`_insertDml = BuildInsertString(tableName);`
`23`	`30`	`_logger = logger;`
	`31`	`+ _hubName = hubName;`
`24`	`32`	`}`
`25`	`33`
`26`	`34`	`private string BuildInsertString(string tableName)`
`@@ -43,6 +51,9 @@ public async Task Send(byte[] message)`
`43`	`51`	`await connection.OpenAsync();`
`44`	`52`
`45`	`53`	`await command.ExecuteNonQueryAsync();`
	`54`	`+`
	`55`	`+ // Record rows written metric`
	`56`	`+ _rowsWrittenCounter.Add(1, new KeyValuePair<string, object?>("signalr.hub", _hubName));`
`46`	`57`	`}`
`47`	`58`	`}`
`48`	`59`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ public SqlStream(SqlServerOptions options, ILogger logger, int streamIndex, stri`
`26`	`26`	`_logger = logger;`
`27`	`27`	`_tracePrefix = tracePrefix;`
`28`	`28`
`29`		`- _sender = new SqlSender(options, logger, tableName);`
	`29`	`+ _sender = new SqlSender(options, logger, tableName, _tracePrefix);`
`30`	`30`	`_receiver = new SqlReceiver(options, logger, tableName, _tracePrefix);`
`31`	`31`	`}`
`32`	`32`