apollo_mempool,apollo_dashboard: add Grafana panels for nonce-gap metrics (#13919)

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: matanl-starkware

crates/apollo_dashboard/resources/dev_grafana.json

@@ -1219,6 +1219,28 @@
           ],
           "extra_params": {}
         },
+        {
+          "title": "Accounts With Nonce Gap",
+          "description": "Number of accounts whose lowest pool nonce exceeds the account nonce, making them unable to provide any transaction for batching",
+          "type": "timeseries",
+          "exprs": [
+            "mempool_accounts_with_gap{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}"
+          ],
+          "extra_params": {
+            "log_query": "\"has a nonce gap\""
+          }
+        },
+        {
+          "title": "Stuck Transactions (Nonce Gap)",
+          "description": "Number of transactions in the pool belonging to accounts whose lowest pool nonce exceeds the account nonce",
+          "type": "timeseries",
+          "exprs": [
+            "mempool_stuck_txs{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}"
+          ],
+          "extra_params": {
+            "log_query": "\"has a nonce gap\""
+          }
+        },
         {
           "title": "Transaction Time Spent In Mempool Until Batched",
           "description": "The time a transaction spends in the mempool until it is batched (5m window)",

crates/apollo_dashboard/src/panels/mempool.rs

@@ -1,9 +1,11 @@
 use apollo_mempool::metrics::{
     LABEL_NAME_DROP_REASON,
+    MEMPOOL_ACCOUNTS_WITH_GAP,
     MEMPOOL_DELAYED_DECLARES_SIZE,
     MEMPOOL_PENDING_QUEUE_SIZE,
     MEMPOOL_POOL_SIZE,
     MEMPOOL_PRIORITY_QUEUE_SIZE,
+    MEMPOOL_STUCK_TXS,
     MEMPOOL_TOTAL_SIZE_BYTES,
     MEMPOOL_TRANSACTIONS_COMMITTED,
     MEMPOOL_TRANSACTIONS_DROPPED,
@@ -97,6 +99,26 @@ fn get_panel_mempool_delayed_declares_size() -> Panel {
         PanelType::TimeSeries,
     )
 }
+fn get_panel_mempool_accounts_with_gap() -> Panel {
+    Panel::new(
+        "Accounts With Nonce Gap",
+        "Number of accounts whose lowest pool nonce exceeds the account nonce, making them unable \
+         to provide any transaction for batching",
+        MEMPOOL_ACCOUNTS_WITH_GAP.get_name_with_filter().to_string(),
+        PanelType::TimeSeries,
+    )
+    .with_log_query("has a nonce gap")
+}
+fn get_panel_mempool_stuck_txs() -> Panel {
+    Panel::new(
+        "Stuck Transactions (Nonce Gap)",
+        "Number of transactions in the pool belonging to accounts whose lowest pool nonce exceeds \
+         the account nonce",
+        MEMPOOL_STUCK_TXS.get_name_with_filter().to_string(),
+        PanelType::TimeSeries,
+    )
+    .with_log_query("has a nonce gap")
+}
 fn get_panel_mempool_transaction_time_spent_until_batched() -> Panel {
     Panel::from_hist(
         &TRANSACTION_TIME_SPENT_UNTIL_BATCHED,
@@ -126,6 +148,8 @@ pub(crate) fn get_mempool_row() -> Row {
             get_panel_mempool_priority_queue_size(),
             get_panel_mempool_pending_queue_size(),
             get_panel_mempool_delayed_declares_size(),
+            get_panel_mempool_accounts_with_gap(),
+            get_panel_mempool_stuck_txs(),
             get_panel_mempool_transaction_time_spent_until_batched(),
             get_panel_mempool_transaction_time_spent_until_committed(),
         ],

crates/apollo_mempool/src/mempool.rs

@@ -942,7 +942,12 @@ impl Mempool {
             if gap_exists {
                 if self.accounts_with_gap.insert(address) {
                     // Newly entered gap: all current pool txs for this account are now stuck.
-                    self.n_stuck_txs += self.tx_pool.n_txs_for_address(address);
+                    let n_stuck = self.tx_pool.n_txs_for_address(address);
+                    self.n_stuck_txs += n_stuck;
+                    warn!(
+                        "Account {address} has a nonce gap; {n_stuck} transaction(s) are now \
+                         stuck."
+                    );
                 }
                 // Stayed in gap: per-tx deltas were already applied at add/remove sites.
             } else {