feat(evm): handle nonce gaps before resubmission (#726)

* fix: Extend non retriable rpc messages

* chore: PR suggestion

* feat: Add nonce sync logic

* chore: Progress

* chore: Improvements

* chore: Improvements

* chore: Improvements

* chore: PR suggestions

* chore: Improvements

---------

Co-authored-by: tirumerla <tirumerla@gmail.com>

Author: zeljkoX

src/constants/evm_transaction.rs

@@ -110,6 +110,40 @@ pub const ALREADY_SUBMITTED_PATTERNS: &[&str] = &[
     "same hash was already imported",
 ];
 
+/// Error message patterns indicating the transaction nonce is ahead of the expected on-chain nonce.
+/// This can be transient (burst ordering: tx N+1 arrives before N) or persistent (counter drift).
+///
+/// Checked **after** `ALREADY_SUBMITTED_PATTERNS` in `classify_submission_error` to avoid
+/// ambiguity. Each entry is a lowercased substring to match against the RPC error message.
+pub const NONCE_TOO_HIGH_PATTERNS: &[&str] = &[
+    "nonce too high",              // Geth, Erigon, Hardhat, Anvil
+    "nonce is too high",           // Geth, Erigon, Hardhat, Anvil
+    "nonce too far in the future", // Besu
+    "exceeds next nonce",          // Nethermind
+    "nonce out of range",          // Arbitrum, Optimism, specialized RPCs
+    "tx-nonce-too-high",           // Certain SaaS RPC providers (e.g. Alchemy/Infura internal)
+];
+
+/// Maximum number of "nonce too high" retries before escalating to a nonce health job.
+/// With ~25s between retries (driven by status checker resend timeout), this means
+/// escalation happens within ~75s — enough time for transient burst ordering to resolve.
+pub const MAX_NONCE_TOO_HIGH_RETRIES: u32 = 3;
+
+/// Maximum number of nonces to scan when detecting gaps between on-chain and local counter.
+/// Gaps beyond this range are logged for operator investigation rather than automated recovery.
+pub const MAX_GAP_SCAN_RANGE: u64 = 100;
+
+/// Metadata key used in `RelayerHealthCheck` to indicate a targeted health action.
+pub const HEALTH_CHECK_ACTION_KEY: &str = "health_check_action";
+
+/// Value for `HEALTH_CHECK_ACTION_KEY` that triggers nonce gap detection and resolution.
+pub const HEALTH_CHECK_ACTION_NONCE_HEALTH: &str = "nonce_health";
+
+/// Optional metadata key carrying a nonce hint for the health action.
+/// When present, `resolve_nonce_gaps` ensures the counter covers at least `hint + 1`
+/// so the scan range includes the hinted nonce. This handles the case where the
+/// counter was reset (e.g., after a restart) but a tx at a higher nonce still exists.
+pub const HEALTH_CHECK_NONCE_HINT_KEY: &str = "nonce_hint";
 /// Checks if a lowercased message matches "known transaction" without matching
 /// "unknown transaction" (substring false positive).
 pub fn matches_known_transaction(msg_lower: &str) -> bool {
@@ -122,3 +156,57 @@ pub fn matches_known_transaction(msg_lower: &str) -> bool {
     }
     false
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_nonce_too_high_patterns_match_expected_strings() {
+        let cases = [
+            "nonce too high",
+            "nonce is too high",
+            "nonce too far in the future",
+            "exceeds next nonce",
+            "nonce out of range",
+        ];
+        for case in &cases {
+            let msg_lower = case.to_lowercase();
+            assert!(
+                NONCE_TOO_HIGH_PATTERNS
+                    .iter()
+                    .any(|p| msg_lower.contains(p)),
+                "Expected NONCE_TOO_HIGH_PATTERNS to match: {case}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_matches_known_transaction_does_not_match_nonce_too_high() {
+        let nonce_too_high_msgs = [
+            "nonce too high",
+            "nonce is too high",
+            "nonce too far in the future",
+            "exceeds next nonce",
+            "nonce out of range",
+        ];
+        for msg in &nonce_too_high_msgs {
+            assert!(
+                !matches_known_transaction(&msg.to_lowercase()),
+                "matches_known_transaction should NOT match nonce-too-high message: {msg}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_matches_known_transaction_matches_known_transaction() {
+        assert!(matches_known_transaction("known transaction"));
+        assert!(matches_known_transaction("already known transaction here"));
+    }
+
+    #[test]
+    fn test_matches_known_transaction_does_not_match_unknown_transaction() {
+        assert!(!matches_known_transaction("unknown transaction"));
+        assert!(!matches_known_transaction("unknown transaction status"));
+    }
+}

src/domain/relayer/evm/evm_relayer.rs

@@ -72,15 +72,15 @@ where
     J: JobProducerTrait + Send + Sync + 'static,
     S: DataSignerTrait + Send + Sync + 'static,
 {
-    relayer: RelayerRepoModel,
-    signer: S,
-    network: EvmNetwork,
-    provider: P,
-    relayer_repository: Arc<RR>,
-    network_repository: Arc<NR>,
-    transaction_repository: Arc<TR>,
-    job_producer: Arc<J>,
-    transaction_counter_service: Arc<TCS>,
+    pub(super) relayer: RelayerRepoModel,
+    pub(super) signer: S,
+    pub(super) network: EvmNetwork,
+    pub(super) provider: P,
+    pub(super) relayer_repository: Arc<RR>,
+    pub(super) network_repository: Arc<NR>,
+    pub(super) transaction_repository: Arc<TR>,
+    pub(super) job_producer: Arc<J>,
+    pub(super) transaction_counter_service: Arc<TCS>,
 }
 
 #[allow(clippy::too_many_arguments)]
@@ -134,50 +134,6 @@ where
         })
     }
 
-    /// Synchronizes the nonce with the blockchain.
-    ///
-    /// # Returns
-    ///
-    /// A `Result` indicating success or a `RelayerError` if the operation fails.
-    #[instrument(
-        level = "debug",
-        skip(self),
-        fields(
-            request_id = ?crate::observability::request_id::get_request_id(),
-            relayer_id = %self.relayer.id,
-        )
-    )]
-    async fn sync_nonce(&self) -> Result<(), RelayerError> {
-        let on_chain_nonce = self
-            .provider
-            .get_transaction_count(&self.relayer.address)
-            .await
-            .map_err(|e| RelayerError::ProviderError(e.to_string()))?;
-
-        let transaction_counter_nonce = self
-            .transaction_counter_service
-            .get()
-            .await
-            .ok()
-            .flatten()
-            .unwrap_or(0);
-
-        let nonce = std::cmp::max(on_chain_nonce, transaction_counter_nonce);
-
-        debug!(
-            relayer_id = %self.relayer.id,
-            on_chain_nonce = %on_chain_nonce,
-            transaction_counter_nonce = %transaction_counter_nonce,
-            "syncing nonce"
-        );
-
-        debug!(nonce = %nonce, "setting nonce for relayer");
-
-        self.transaction_counter_service.set(nonce).await?;
-
-        Ok(())
-    }
-
     /// Validates the RPC connection to the blockchain provider.
     ///
     /// # Returns

src/domain/relayer/evm/mod.rs

@@ -3,6 +3,7 @@
 /// the `evm_relayer` submodule which contains the core logic for
 /// relaying transactions and events between different EVM networks.
 mod evm_relayer;
+mod nonce;
 mod rpc_utils;
 mod validations;