Add INFO COMMANDSTATS tracking with per-command success rates (#1702)

* Add INFO COMMANDSTATS tracking with per-command success rates

Implements Redis-compatible INFO COMMANDSTATS support, gated behind
--commandstats-monitor config flag. Tracks per-command calls, failed_calls,
and rejected_calls using lightweight counter increments (no Stopwatch).

Key design decisions:
- Array-indexed by RespCommand enum for O(1) access
- Per-session counters (single-writer, no locking on hot path)
- On-demand aggregation from active sessions + disposed session history
- No latency tracking (usec=0) to avoid Stopwatch overhead (~4.4x perf hit)
- Monitor dispose handles case where Start() was never called (EmbeddedServer)

Benchmark results (100 PINGs, .NET 10):
  Disabled: 1.687 us | Enabled: 1.820 us | Overhead: ~7.8%
  SET/GET overhead: 0-4% (within noise for real workloads)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Fix double-counting bug in PopulateCommandStatsInfo and remove stale latency references

When MetricsSamplingFrequency > 0, globalCommandStats already includes
history from disposed sessions. PopulateCommandStatsInfo was adding
historyCommandStats on top, double-counting disposed session stats.

Also removed stale 'latency' references from doc comments and help text
since per-command latency tracking was removed to avoid Stopwatch overhead.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Address PR review: simplify error detection, use canonical command names, fix tests

- Remove bufferFlushedDuringCommand field and pointer-based error detection;
  rely solely on commandErrorWritten flag set by WriteError/AbortWithErrorMessage
- Use RespCommandsInfo.GetRespCommandName() for canonical Redis command names
  instead of enum ToString with string replacement
- Switch most tests to metricsSamplingFreq: 0 (on-demand aggregation) to remove
  Thread.Sleep delays; keep one test with periodic sampling for coverage
- Fix CommandStatsFailedCallsTest to use SETRANGE with invalid offset (goes
  through AbortWithErrorMessage) instead of WRONGTYPE which bypasses the flag

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Include LatencyMonitor in monitor creation condition

GarnetLatencyMetricsSession dereferences monitor.monitor_iterations, but
StoreWrapper only created the monitor when MetricsSamplingFrequency > 0 or
CommandStatsMonitor was enabled. With --latency-monitor alone and no
sampling frequency, this would cause a NullReferenceException at session
creation. Add LatencyMonitor to the condition so the monitor is always
created when latency tracking is enabled.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Update REspInfo test

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Mathos1432

libs/common/Metrics/InfoMetricsType.cs

@@ -86,6 +86,10 @@ public enum InfoMetricsType : byte
         /// Scan and return distribution of in-memory portion of hybrid logs for main store and object store
         /// </summary>
         HLOGSCAN,
+        /// <summary>
+        /// Per-command usage statistics (calls, failures, rejections)
+        /// </summary>
+        COMMANDSTATS,
     }
 
     /// <summary>

libs/host/Configuration/Options.cs

@@ -351,6 +351,10 @@ internal sealed class Options : ICloneable
         [Option("latency-monitor", Required = false, HelpText = "Track latency of various events.")]
         public bool? LatencyMonitor { get; set; }
 
+        [OptionValidation]
+        [Option("commandstats-monitor", Required = false, HelpText = "Track per-command usage statistics (calls, failures, rejections). Exposed via INFO COMMANDSTATS.")]
+        public bool? CommandStatsMonitor { get; set; }
+
         [IntRangeValidation(0, int.MaxValue)]
         [Option("slowlog-log-slower-than", Required = false, HelpText = "Threshold (microseconds) for logging command in the slow log. 0 to disable.")]
         public int SlowLogThreshold { get; set; }
@@ -916,6 +920,7 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno
                     ServerCertificateRequired.GetValueOrDefault(),
                     logger: logger) : null,
                 LatencyMonitor = LatencyMonitor.GetValueOrDefault(),
+                CommandStatsMonitor = CommandStatsMonitor.GetValueOrDefault(),
                 SlowLogThreshold = SlowLogThreshold,
                 SlowLogMaxEntries = SlowLogMaxEntries,
                 MetricsSamplingFrequency = MetricsSamplingFrequency,

libs/host/Configuration/Redis/RedisOptions.cs

@@ -83,6 +83,9 @@ internal class RedisOptions
         [RedisOption("latency-tracking", nameof(Options.LatencyMonitor))]
         public Option<RedisBoolean> LatencyTracking { get; set; }
 
+        [RedisOption("commandstats-tracking", nameof(Options.CommandStatsMonitor))]
+        public Option<RedisBoolean> CommandStatsTracking { get; set; }
+
         [RedisOption("loglevel", nameof(Options.LogLevel))]
         public Option<RedisLogLevel> LogLevel { get; set; }
 

libs/host/defaults.conf

@@ -254,6 +254,9 @@
 	/* Track latency of various events. */
 	"LatencyMonitor" : false,
 
+	/* Track per-command usage statistics (calls, failures, rejections). Exposed via INFO COMMANDSTATS. */
+	"CommandStatsMonitor" : false,
+
 	/* Threshold (microseconds) for logging command in the slow log. 0 to disable. */
 	"SlowLogThreshold": 0,
 

libs/server/Metrics/CommandStats.cs

@@ -0,0 +1,127 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+using System.Runtime.CompilerServices;
+
+namespace Garnet.server
+{
+    /// <summary>
+    /// Per-command statistics entry tracking calls, failures, and rejections.
+    /// Follows the Redis COMMANDSTATS convention.
+    /// </summary>
+    public struct CommandStatsEntry
+    {
+        /// <summary>
+        /// Total number of times this command was called.
+        /// </summary>
+        public ulong Calls;
+
+        /// <summary>
+        /// Total number of times this command failed (returned an error response).
+        /// </summary>
+        public ulong FailedCalls;
+
+        /// <summary>
+        /// Total number of times this command was rejected before execution (e.g., ACL denied, OOM).
+        /// </summary>
+        public ulong RejectedCalls;
+    }
+
+    /// <summary>
+    /// Tracks per-command usage statistics for built-in commands.
+    /// Array-indexed by RespCommand enum value for O(1) access.
+    /// Each session owns its own instance (single-writer, no locking needed).
+    /// </summary>
+    public class CommandStats
+    {
+        /// <summary>
+        /// Number of entries in the stats array, sized to hold all valid RespCommand values.
+        /// </summary>
+        internal static readonly int EntryCount = (int)RespCommandExtensions.LastValidCommand + 1;
+
+        /// <summary>
+        /// Per-command statistics entries indexed by (int)RespCommand.
+        /// </summary>
+        internal CommandStatsEntry[] entries;
+
+        /// <summary>
+        /// Creates a new CommandStats instance with zeroed entries.
+        /// </summary>
+        public CommandStats()
+        {
+            entries = new CommandStatsEntry[EntryCount];
+        }
+
+        /// <summary>
+        /// Increment the calls counter for the given command.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void IncrementCalls(RespCommand cmd)
+        {
+            ushort idx = (ushort)cmd;
+            if (idx < entries.Length)
+                entries[idx].Calls++;
+        }
+
+        /// <summary>
+        /// Increment the failed calls counter for the given command.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void IncrementFailed(RespCommand cmd)
+        {
+            ushort idx = (ushort)cmd;
+            if (idx < entries.Length)
+                entries[idx].FailedCalls++;
+        }
+
+        /// <summary>
+        /// Increment the rejected calls counter for the given command.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void IncrementRejected(RespCommand cmd)
+        {
+            ushort idx = (ushort)cmd;
+            if (idx < entries.Length)
+                entries[idx].RejectedCalls++;
+        }
+
+        /// <summary>
+        /// Get the stats entry for the given command.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public CommandStatsEntry GetEntry(RespCommand cmd)
+        {
+            ushort idx = (ushort)cmd;
+            if (idx < entries.Length)
+                return entries[idx];
+            return default;
+        }
+
+        /// <summary>
+        /// Add another CommandStats instance into this one (for aggregation).
+        /// </summary>
+        internal void Add(CommandStats other)
+        {
+            if (other?.entries == null)
+                return;
+
+            int len = Math.Min(entries.Length, other.entries.Length);
+            for (int i = 0; i < len; i++)
+            {
+                entries[i].Calls += other.entries[i].Calls;
+                entries[i].FailedCalls += other.entries[i].FailedCalls;
+                entries[i].RejectedCalls += other.entries[i].RejectedCalls;
+            }
+        }
+
+        /// <summary>
+        /// Reset all entries to zero.
+        /// </summary>
+        internal void Reset()
+        {
+            Array.Clear(entries, 0, entries.Length);
+        }
+
+    }
+}

libs/server/Metrics/GarnetServerMetrics.cs

@@ -35,7 +35,17 @@ internal struct GarnetServerMetrics
         /// </summary>
         public readonly GarnetLatencyMetrics globalLatencyMetrics;
 
-        public GarnetServerMetrics(bool trackStats, bool trackLatency, GarnetServerMonitor monitor)
+        /// <summary>
+        /// Global per-command usage statistics (calls, failures, rejections).
+        /// </summary>
+        public CommandStats globalCommandStats;
+
+        /// <summary>
+        /// History of per-command usage statistics from disposed sessions.
+        /// </summary>
+        public CommandStats historyCommandStats;
+
+        public GarnetServerMetrics(bool trackStats, bool trackLatency, bool trackCommandStats, GarnetServerMonitor monitor)
         {
             total_connections_received = 0;
             total_connections_disposed = 0;
@@ -49,6 +59,9 @@ public GarnetServerMetrics(bool trackStats, bool trackLatency, GarnetServerMonit
             historySessionMetrics = trackStats ? new GarnetSessionMetrics() : null;
 
             globalLatencyMetrics = trackLatency ? new() : null;
+
+            globalCommandStats = trackCommandStats ? new CommandStats() : null;
+            historyCommandStats = trackCommandStats ? new CommandStats() : null;
         }
 
         public void Dispose()

libs/server/Metrics/GarnetServerMonitor.cs

@@ -20,7 +20,7 @@ internal enum EventType : byte
     internal sealed class GarnetServerMonitor
     {
         public readonly Dictionary<InfoMetricsType, bool>
-            resetEventFlags = GarnetInfoMetrics.DefaultInfo.ToDictionary(x => x, y => false);
+            resetEventFlags = Enum.GetValues<InfoMetricsType>().ToDictionary(x => x, y => false);
 
         public readonly Dictionary<LatencyMetricsType, bool>
             resetLatencyMetrics = GarnetLatencyMetrics.defaultLatencyTypes.ToDictionary(x => x, y => false);
@@ -33,17 +33,20 @@ public readonly Dictionary<LatencyMetricsType, bool>
 
         GarnetServerMetrics globalMetrics;
         readonly GarnetSessionMetrics accSessionMetrics;
+        readonly CommandStats accCommandStats;
         private ulong instant_input_net_bytes;
         private ulong instant_output_net_bytes;
         private ulong instant_commands_processed;
 
         readonly CancellationTokenSource cts = new();
-        readonly ManualResetEvent done = new(false);
+        readonly ManualResetEvent done = new(true);
 
         readonly ILogger logger;
 
         public GarnetServerMetrics GlobalMetrics => globalMetrics;
 
+        internal IGarnetServer[] Servers => servers;
+
         SingleWriterMultiReaderLock rwLock = new();
 
         public GarnetServerMonitor(StoreWrapper storeWrapper, GarnetServerOptions opts, IGarnetServer[] servers, ILogger logger = null)
@@ -58,9 +61,10 @@ public GarnetServerMonitor(StoreWrapper storeWrapper, GarnetServerOptions opts,
             instant_input_net_bytes = 0;
             instant_output_net_bytes = 0;
             instant_commands_processed = 0;
-            globalMetrics = new(true, opts.LatencyMonitor, this);
+            globalMetrics = new(true, opts.LatencyMonitor, opts.CommandStatsMonitor, this);
 
             accSessionMetrics = new GarnetSessionMetrics();
+            accCommandStats = opts.CommandStatsMonitor ? new CommandStats() : null;
         }
 
         public void Dispose()
@@ -74,16 +78,23 @@ public void Dispose()
 
         public void Start()
         {
-            Task.Run(() => MainMonitorTask(cts.Token));
+            // Only run the periodic sampling task if a sampling frequency is configured.
+            // The monitor may be created solely for command stats history (without periodic sampling).
+            if (monitorSamplingFrequency > TimeSpan.Zero)
+            {
+                done.Reset();
+                Task.Run(() => MainMonitorTask(cts.Token));
+            }
         }
 
-        public void AddMetricsHistorySessionDispose(GarnetSessionMetrics currSessionMetrics, GarnetLatencyMetricsSession currLatencyMetrics)
+        public void AddMetricsHistorySessionDispose(GarnetSessionMetrics currSessionMetrics, GarnetLatencyMetricsSession currLatencyMetrics, CommandStats currCommandStats = null)
         {
             rwLock.WriteLock();
             try
             {
                 if (currSessionMetrics != null) globalMetrics.historySessionMetrics.Add(currSessionMetrics);
                 if (currLatencyMetrics != null) globalMetrics.globalLatencyMetrics.Merge(currLatencyMetrics);
+                if (currCommandStats != null) globalMetrics.historyCommandStats?.Add(currCommandStats);
                 currLatencyMetrics?.Return();
             }
             finally { rwLock.WriteUnlock(); }
@@ -133,6 +144,12 @@ private void AddCurrentServerStats(IGarnetServer server)
                 // Accumulate session metrics
                 accSessionMetrics.Add(session.GetSessionMetrics);
 
+                // Accumulate command stats if enabled
+                if (accCommandStats != null)
+                {
+                    accCommandStats.Add(session.GetCommandStats);
+                }
+
                 // Accumulate latency metrics if latency monitor is enabled
                 if (opts.LatencyMonitor)
                 {
@@ -154,6 +171,12 @@ private void AddCurrentServerStats(IGarnetServer server)
             // Add accumulated session metrics for this iteration
             globalMetrics.globalSessionMetrics.Add(accSessionMetrics);
 
+            // Reset global command stats and add accumulated for this iteration
+            if (accCommandStats != null)
+            {
+                globalMetrics.globalCommandStats.Reset();
+                globalMetrics.globalCommandStats.Add(accCommandStats);
+            }
         }
 
         private void CleanupGlobalStats()
@@ -189,6 +212,24 @@ private void CleanupGlobalStats()
 
                 resetEventFlags[InfoMetricsType.STATS] = false;
             }
+
+            if (resetEventFlags.TryGetValue(InfoMetricsType.COMMANDSTATS, out bool resetCommandStats) && resetCommandStats)
+            {
+                logger?.LogInformation("Resetting command stats");
+                globalMetrics.globalCommandStats?.Reset();
+                globalMetrics.historyCommandStats?.Reset();
+
+                foreach (var garnetServer in servers.Cast<GarnetServerBase>())
+                {
+                    var sessions = garnetServer.ActiveConsumers();
+                    foreach (var s in sessions)
+                    {
+                        ((RespServerSession)s).GetCommandStats?.Reset();
+                    }
+                }
+
+                resetEventFlags[InfoMetricsType.COMMANDSTATS] = false;
+            }
         }
 
         private void CleanupGlobalLatencyMetrics()
@@ -283,6 +324,13 @@ void ResetAndAddGlobalHistory()
                 accSessionMetrics.Reset();
                 // Add session metrics history in accumulator
                 accSessionMetrics.Add(globalMetrics.historySessionMetrics);
+
+                // Reset command stats accumulator and add history
+                if (accCommandStats != null)
+                {
+                    accCommandStats.Reset();
+                    accCommandStats.Add(globalMetrics.historyCommandStats);
+                }
             }
 
             void ResetLatencySessionMetrics()