diff --git a/src/constants/stellar_transaction.rs b/src/constants/stellar_transaction.rs index 8097ee344..1c1a79b60 100644 --- a/src/constants/stellar_transaction.rs +++ b/src/constants/stellar_transaction.rs @@ -72,7 +72,16 @@ pub const STELLAR_MAX_STUCK_TRANSACTION_LIFETIME_MINUTES: i64 = 15; pub const STELLAR_RESUBMIT_BASE_INTERVAL_SECONDS: i64 = 10; /// Maximum number of times a Stellar submission may be retried after an insufficient-fee error. -pub const STELLAR_INSUFFICIENT_FEE_MAX_RETRIES: u32 = 2; +/// Raised to cover multiple fast resubmits across short-lived fee spikes. +pub const STELLAR_INSUFFICIENT_FEE_MAX_RETRIES: u32 = 6; + +/// Base delay for fast Stellar resubmits, in seconds. +/// Five seconds is approximately one Stellar ledger close time (`STELLAR_LEDGER_TIME_SECONDS`). +pub const STELLAR_FAST_RESUBMIT_BASE_DELAY_SECONDS: i64 = 5; + +/// Maximum TRY_AGAIN_LATER rejections that use the fast-resubmit window. +/// After the first three fast attempts, the status-checker backoff ladder owns rescue. +pub const STELLAR_TRY_AGAIN_LATER_FAST_RETRIES: u32 = 3; /// Maximum resubmit interval (seconds) to cap the resubmission backoff. /// Prevents excessively long gaps between resubmissions. diff --git a/src/domain/transaction/stellar/submit.rs b/src/domain/transaction/stellar/submit.rs index 19e42e1f2..c9e96213a 100644 --- a/src/domain/transaction/stellar/submit.rs +++ b/src/domain/transaction/stellar/submit.rs @@ -11,7 +11,11 @@ use super::{ StellarRelayerTransaction, }; use crate::{ - constants::STELLAR_INSUFFICIENT_FEE_MAX_RETRIES, + constants::{ + STELLAR_FAST_RESUBMIT_BASE_DELAY_SECONDS, STELLAR_INSUFFICIENT_FEE_MAX_RETRIES, + STELLAR_TRY_AGAIN_LATER_FAST_RETRIES, + }, + domain::transaction::stellar::prepare::common::send_submit_transaction_job, jobs::JobProducerTrait, metrics::{STELLAR_SUBMISSION_FAILURES, TRANSACTIONS_INSUFFICIENT_FEE}, models::{ @@ -88,10 +92,14 @@ where /// Handles status codes: /// - PENDING: Transaction accepted for processing /// - DUPLICATE: Transaction already submitted (treat as success) - /// - TRY_AGAIN_LATER: Network congested but tx is valid — update sent_at and return Ok - /// (status checker will retry with exponential backoff) - /// - ERROR: Transaction validation failed, mark as failed, except for insufficient fee errors - /// (insufficient fee errors are treated as TRY_AGAIN_LATER) + /// - TRY_AGAIN_LATER: Network congested but tx is valid; update sent_at and schedule a + /// fast delayed resubmit (5s × attempt) for the first 3 rejections + /// - ERROR: Transaction validation failed, except insufficient fee schedules the same fast + /// delayed resubmit through the insufficient-fee retry cap (6) + /// + /// The status checker remains the fallback rescue. Each fast attempt refreshes sent_at before + /// enqueueing, so the ladder's ≥10s time-since-last-submit gate stays closed while the + /// 5s-spaced fast attempts are active. async fn submit_core( &self, tx: TransactionRepoModel, @@ -162,15 +170,27 @@ where // Network is temporarily congested — the transaction is valid but the // node's queue is full. Atomically update sent_at and increment // try_again_later_retries so the status checker's backoff gate measures - // time since this attempt. Return Ok to keep the transaction alive. - // The status checker will handle retries: - // - Submitted txs: resubmitted with exponential backoff - // - Sent txs: re-enqueued via handle_sent_state + // time since this attempt. Return Ok to keep the transaction alive; + // the fast path handles the bounded early retries, and the status + // checker remains the fallback rescue. let updated_tx = self .transaction_repository() .record_stellar_try_again_later_retry(tx.id.clone(), Utc::now().to_rfc3339()) .await?; + // The repo returns the transaction unchanged if it reached a final state + // between this submission and the record call (e.g. a racing job finalized + // it). Nothing left to rescue — skip metrics and scheduling. + if is_final_state(&updated_tx.status) { + debug!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + status = ?updated_tx.status, + "transaction reached final state during retry recording; skipping fast resubmit" + ); + return Ok(updated_tx); + } + let retries = updated_tx .metadata .as_ref() @@ -183,13 +203,44 @@ where .inc(); } - debug!( - tx_id = %tx.id, - relayer_id = %tx.relayer_id, - status = ?tx.status, - try_again_later_retries = retries, - "TRY_AGAIN_LATER — status checker will retry" - ); + if retries <= STELLAR_TRY_AGAIN_LATER_FAST_RETRIES { + let delay_seconds = + STELLAR_FAST_RESUBMIT_BASE_DELAY_SECONDS * i64::from(retries); + + info!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + status = ?updated_tx.status, + try_again_later_retries = retries, + delay_seconds, + "enqueueing fast resubmit after TRY_AGAIN_LATER" + ); + + if let Err(error) = send_submit_transaction_job( + self.job_producer(), + &updated_tx, + Some(delay_seconds), + ) + .await + { + warn!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + error = %error, + try_again_later_retries = retries, + delay_seconds, + "failed to enqueue fast resubmit after TRY_AGAIN_LATER" + ); + } + } else { + debug!( + tx_id = %tx.id, + relayer_id = %tx.relayer_id, + status = ?tx.status, + try_again_later_retries = retries, + "TRY_AGAIN_LATER — status checker will retry" + ); + } Ok(updated_tx) } "ERROR" => { @@ -225,14 +276,6 @@ where ))); } - debug!( - tx_id = %tx.id, - relayer_id = %tx.relayer_id, - status = ?tx.status, - insufficient_fee_retries = meta.insufficient_fee_retries, - result_code = decoded_result_code.as_deref().unwrap_or("Unknown"), - "ERROR with insufficient fee — status checker will retry" - ); // Atomically sets `sent_at` and increments Stellar insufficient-fee retries. let updated_tx = self .transaction_repository() @@ -241,6 +284,67 @@ where Utc::now().to_rfc3339(), ) .await?; + + // The repo returns the transaction unchanged if it reached a final state + // between this submission and the record call (e.g. a racing job finalized + // it). Nothing left to rescue — skip metrics and scheduling. + if is_final_state(&updated_tx.status) { + debug!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + status = ?updated_tx.status, + "transaction reached final state during retry recording; skipping fast resubmit" + ); + return Ok(updated_tx); + } + + let retries = updated_tx + .metadata + .as_ref() + .map_or(0, |metadata| metadata.insufficient_fee_retries); + + // This can only happen when a concurrent submission increments the counter + // between our read and record. That winner already scheduled the retry; skip + // this enqueue so the next rejection terminates normally at the pre-record cap. + if retries > STELLAR_INSUFFICIENT_FEE_MAX_RETRIES { + warn!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + insufficient_fee_retries = retries, + "post-record retry count exceeds cap; skipping fast resubmit - concurrent submission detected" + ); + return Ok(updated_tx); + } + + let delay_seconds = + STELLAR_FAST_RESUBMIT_BASE_DELAY_SECONDS * i64::from(retries); + + info!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + status = ?updated_tx.status, + insufficient_fee_retries = retries, + delay_seconds, + result_code = decoded_result_code.as_deref().unwrap_or("Unknown"), + "enqueueing fast resubmit after insufficient fee" + ); + + if let Err(error) = send_submit_transaction_job( + self.job_producer(), + &updated_tx, + Some(delay_seconds), + ) + .await + { + warn!( + tx_id = %updated_tx.id, + relayer_id = %updated_tx.relayer_id, + error = %error, + insufficient_fee_retries = retries, + delay_seconds, + "failed to enqueue fast resubmit after insufficient fee" + ); + } return Ok(updated_tx); } STELLAR_SUBMISSION_FAILURES @@ -1006,6 +1110,13 @@ mod tests { Ok::<_, RepositoryError>(tx) }); + mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(|_, scheduled_on| scheduled_on.is_some()) + .times(1) + .returning(|_, _| Box::pin(async { Ok(()) })); + let handler = make_stellar_tx_handler(relayer.clone(), mocks); let mut tx = create_test_transaction(&relayer.id); tx.status = TransactionStatus::Sent; @@ -1016,7 +1127,7 @@ mod tests { let res = handler.submit_transaction_impl(tx).await; - // Transaction stays in Sent — status checker will re-enqueue submission + // Transaction stays in Sent and gets a delayed fast resubmit. let returned_tx = res.unwrap(); assert_eq!(returned_tx.status, TransactionStatus::Sent); } @@ -1059,6 +1170,13 @@ mod tests { Ok::<_, RepositoryError>(tx) }); + submit_mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(|_, scheduled_on| scheduled_on.is_some()) + .times(1) + .returning(|_, _| Box::pin(async { Ok(()) })); + let submit_handler = make_stellar_tx_handler(relayer.clone(), submit_mocks); let mut sent_tx = create_test_transaction(&relayer.id); sent_tx.status = TransactionStatus::Sent; @@ -1140,6 +1258,13 @@ mod tests { Ok::<_, RepositoryError>(tx) }); + mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(|_, scheduled_on| scheduled_on.is_some()) + .times(1) + .returning(|_, _| Box::pin(async { Ok(()) })); + let handler = make_stellar_tx_handler(relayer.clone(), mocks); let mut tx = create_test_transaction(&relayer.id); tx.status = TransactionStatus::Submitted; // Already submitted (resubmission path) @@ -1150,11 +1275,125 @@ mod tests { let res = handler.submit_transaction_impl(tx).await; - // Should succeed without marking as failed — status checker will retry + // Should succeed without marking as failed and get a delayed fast resubmit. let returned_tx = res.unwrap(); assert_eq!(returned_tx.status, TransactionStatus::Submitted); } + #[tokio::test] + async fn submit_transaction_try_again_later_fast_window_closes() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + + let response = create_send_tx_response( + "TRY_AGAIN_LATER", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_try_again_later_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 0, + try_again_later_retries: 4, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + let res = handler.submit_transaction_impl(tx).await; + + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Sent); + assert_eq!( + returned_tx + .metadata + .as_ref() + .map(|metadata| metadata.try_again_later_retries), + Some(4) + ); + } + + #[tokio::test] + async fn submit_transaction_try_again_later_final_state_skips_enqueue() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + + let response = create_send_tx_response( + "TRY_AGAIN_LATER", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_try_again_later_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Confirmed; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 0, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }) + .times(1); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 0, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + let res = handler.submit_core(tx).await; + + assert!(res.is_ok()); + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Confirmed); + } + #[tokio::test] async fn submit_transaction_error_status_fails() { let relayer = create_test_relayer(); @@ -1260,6 +1499,13 @@ mod tests { }) .times(1); + mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(|_, scheduled_on| scheduled_on.is_some()) + .times(1) + .returning(|_, _| Box::pin(async { Ok(()) })); + let handler = make_stellar_tx_handler(relayer.clone(), mocks); let mut tx = create_test_transaction(&relayer.id); tx.status = TransactionStatus::Sent; @@ -1282,10 +1528,349 @@ mod tests { ); } + #[tokio::test] + async fn submit_transaction_insufficient_fee_escalates_delay() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + let before_submit = std::sync::Arc::new(std::sync::atomic::AtomicI64::new(0)); + let before_submit_for_mock = before_submit.clone(); + let expected_delay = STELLAR_FAST_RESUBMIT_BASE_DELAY_SECONDS * 3; + + let mut response = create_send_tx_response( + "ERROR", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + response.error_result_xdr = Some("AAAAAAAAY/n////3AAAAAA==".to_string()); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_insufficient_fee_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 3, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }) + .times(1); + + mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(move |_, scheduled_on| { + let before_submit = + before_submit_for_mock.load(std::sync::atomic::Ordering::SeqCst); + let now = Utc::now().timestamp(); + scheduled_on.is_some_and(|scheduled_on| { + scheduled_on >= before_submit + expected_delay - 1 + && scheduled_on <= now + expected_delay + 1 + }) + }) + .times(1) + .returning(|_, _| Box::pin(async { Ok(()) })); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + before_submit.store(Utc::now().timestamp(), std::sync::atomic::Ordering::SeqCst); + let res = handler.submit_transaction_impl(tx).await; + + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Sent); + assert_eq!( + returned_tx + .metadata + .as_ref() + .map(|metadata| metadata.insufficient_fee_retries), + Some(3) + ); + } + + #[tokio::test] + async fn submit_transaction_insufficient_fee_post_record_over_cap_skips_enqueue() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + + let mut response = create_send_tx_response( + "ERROR", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + response.error_result_xdr = Some("AAAAAAAAY/n////3AAAAAA==".to_string()); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_insufficient_fee_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: STELLAR_INSUFFICIENT_FEE_MAX_RETRIES + 1, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }) + .times(1); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + let res = handler.submit_core(tx).await; + + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Sent); + assert_eq!( + returned_tx + .metadata + .as_ref() + .map(|metadata| metadata.insufficient_fee_retries), + Some(STELLAR_INSUFFICIENT_FEE_MAX_RETRIES + 1) + ); + } + + #[tokio::test] + async fn submit_transaction_insufficient_fee_final_state_skips_enqueue() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + + let mut response = create_send_tx_response( + "ERROR", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + response.error_result_xdr = Some("AAAAAAAAY/n////3AAAAAA==".to_string()); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_insufficient_fee_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Confirmed; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 0, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }) + .times(1); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 0, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + let res = handler.submit_core(tx).await; + + assert!(res.is_ok()); + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Confirmed); + } + + #[tokio::test] + async fn submit_transaction_fast_resubmit_enqueue_failure_is_swallowed() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + + let mut response = create_send_tx_response( + "ERROR", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + response.error_result_xdr = Some("AAAAAAAAY/n////3AAAAAA==".to_string()); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_insufficient_fee_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 1, + try_again_later_retries: 0, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }) + .times(1); + + mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(|_, scheduled_on| scheduled_on.is_some()) + .times(1) + .returning(|_, _| { + Box::pin(async { + Err(crate::jobs::JobProducerError::QueueError( + "queue unavailable".to_string(), + )) + }) + }); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + let res = handler.submit_transaction_impl(tx).await; + + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Sent); + assert_eq!( + returned_tx + .metadata + .as_ref() + .map(|metadata| metadata.insufficient_fee_retries), + Some(1) + ); + } + + #[tokio::test] + async fn submit_transaction_try_again_later_enqueue_failure_is_swallowed() { + let relayer = create_test_relayer(); + let mut mocks = default_test_mocks(); + + let response = create_send_tx_response( + "TRY_AGAIN_LATER", + "0101010101010101010101010101010101010101010101010101010101010101", + ); + mocks + .provider + .expect_send_transaction_with_status() + .returning(move |_| { + let r = response.clone(); + Box::pin(async move { Ok(r) }) + }); + + mocks + .tx_repo + .expect_record_stellar_try_again_later_retry() + .withf(|id, sent_at| id == "tx-1" && !sent_at.is_empty()) + .returning(|id, _| { + let mut tx = create_test_transaction("relayer-1"); + tx.id = id; + tx.status = TransactionStatus::Sent; + tx.metadata = Some(TransactionMetadata { + consecutive_failures: 0, + total_failures: 0, + insufficient_fee_retries: 0, + try_again_later_retries: 1, + nonce_too_high_retries: 0, + }); + Ok::<_, RepositoryError>(tx) + }) + .times(1); + + mocks + .job_producer + .expect_produce_submit_transaction_job() + .withf(|_, scheduled_on| scheduled_on.is_some()) + .times(1) + .returning(|_, _| { + Box::pin(async { + Err(crate::jobs::JobProducerError::QueueError( + "queue unavailable".to_string(), + )) + }) + }); + + let handler = make_stellar_tx_handler(relayer.clone(), mocks); + let mut tx = create_test_transaction(&relayer.id); + tx.status = TransactionStatus::Sent; + if let NetworkTransactionData::Stellar(ref mut data) = tx.network_data { + data.signatures.push(dummy_signature()); + data.signed_envelope_xdr = Some(create_signed_xdr(TEST_PK, TEST_PK_2)); + } + + let res = handler.submit_transaction_impl(tx).await; + + let returned_tx = res.unwrap(); + assert_eq!(returned_tx.status, TransactionStatus::Sent); + assert_eq!( + returned_tx + .metadata + .as_ref() + .map(|metadata| metadata.try_again_later_retries), + Some(1) + ); + } + #[tokio::test] async fn submit_transaction_insufficient_fee_exceeding_retry_limit_fails() { let relayer = create_test_relayer(); let mut mocks = default_test_mocks(); + let retry_limit_reason = format!( + "insufficient fee retry limit exceeded ({STELLAR_INSUFFICIENT_FEE_MAX_RETRIES})" + ); + let retry_limit_reason_for_mock = retry_limit_reason.clone(); let mut response = create_send_tx_response( "ERROR", @@ -1303,11 +1888,12 @@ mod tests { mocks .tx_repo .expect_partial_update() - .withf(|_, upd| { + .withf(move |_, upd| { upd.status == Some(TransactionStatus::Failed) - && upd.status_reason.as_ref().is_some_and(|reason| { - reason.contains("insufficient fee retry limit exceeded (2)") - }) + && upd + .status_reason + .as_ref() + .is_some_and(|reason| reason.contains(&retry_limit_reason_for_mock)) }) .returning(|id, upd| { let mut tx = create_test_transaction("relayer-1"); @@ -1351,11 +1937,10 @@ mod tests { let failed_tx = res.unwrap(); assert_eq!(failed_tx.status, TransactionStatus::Failed); - assert!( - failed_tx.status_reason.as_ref().is_some_and( - |reason| reason.contains("insufficient fee retry limit exceeded (2)") - ) - ); + assert!(failed_tx + .status_reason + .as_ref() + .is_some_and(|reason| reason.contains(&retry_limit_reason))); } #[tokio::test] diff --git a/src/queues/redis/queue.rs b/src/queues/redis/queue.rs index 6a2be40c3..32167d1d5 100644 --- a/src/queues/redis/queue.rs +++ b/src/queues/redis/queue.rs @@ -227,8 +227,8 @@ impl Queue { .unwrap_or_default(); // Queue configurations: - // - High-frequency: transaction_status (critical path) - // - Low-frequency: request, submission, notifications, health checks, swaps + // - High-frequency: transaction_submission, transaction_status (critical path) + // - Low-frequency: request, notifications, health checks, swaps let high_frequency = QueueConfig::high_frequency(); let low_frequency = QueueConfig::low_frequency(); @@ -241,7 +241,9 @@ impl Queue { transaction_submission_queue: Self::storage( &format!("{redis_key_prefix}transaction_submission_queue"), conn_tx_submit, - low_frequency.clone(), // scheduling not used + // Stellar fast resubmit schedules 5s*attempt delayed submit jobs here. + // A 20s sweep quantizes those retries and lets the status checker double-fire. + high_frequency.clone(), ), transaction_status_queue: Self::storage( &format!("{redis_key_prefix}transaction_status_queue"), diff --git a/testing/wiremock/.gitignore b/testing/wiremock/.gitignore new file mode 100644 index 000000000..69223617f --- /dev/null +++ b/testing/wiremock/.gitignore @@ -0,0 +1,4 @@ +e2e/env.* +e2e/*.keystore +e2e/**/*.keystore +e2e/generated-relayers/ diff --git a/testing/wiremock/README.md b/testing/wiremock/README.md index 17cc39925..e4c215834 100644 --- a/testing/wiremock/README.md +++ b/testing/wiremock/README.md @@ -81,9 +81,89 @@ All scenarios fire **exactly once**, then revert to proxying real traffic. |--------|-------------| | `trigger-try-again-later.sh` | Arm the TRY_AGAIN_LATER scenario | | `trigger-insufficient-fee.sh` | Arm the insufficient fee scenario | +| `e2e-fast-resubmit.sh` | Run the Stellar fast-resubmit E2E matrix | | `check-scenarios.sh` | Show scenario states and recent requests | | `reset-all.sh` | Reset all scenarios and clear request log | +## E2E fast-resubmit verification + +Run the Stellar fast-resubmit matrix from the repo root: + +```bash +./testing/wiremock/scripts/e2e-fast-resubmit.sh all +./testing/wiremock/scripts/e2e-fast-resubmit.sh -o /tmp/fast-resubmit-artifacts all +``` + +Run one scenario by name: + +```bash +./testing/wiremock/scripts/e2e-fast-resubmit.sh if-once +./testing/wiremock/scripts/e2e-fast-resubmit.sh if-escalation +./testing/wiremock/scripts/e2e-fast-resubmit.sh if-overcap +./testing/wiremock/scripts/e2e-fast-resubmit.sh tal-window +./testing/wiremock/scripts/e2e-fast-resubmit.sh tal-window-closes +``` + +The harness verifies the durable regression cases for Stellar fast resubmit on testnet: + +| Scenario | Injected responses | Expected behavior | +|----------|--------------------|-------------------| +| `if-once` | 1 insufficient-fee response | One fast retry after about 5s, then submission proceeds | +| `if-escalation` | 3 insufficient-fee responses | Fast retries after about 5s, 10s, and 15s | +| `if-overcap` | 7 insufficient-fee responses | Six fast retries, then failure at the retry cap | +| `tal-window` | 3 TRY_AGAIN_LATER responses | Fast retries after about 5s, 10s, and 15s | +| `tal-window-closes` | 4 TRY_AGAIN_LATER responses | Three fast retries, then status-checker handoff | + +The one-shot WireMock mappings remain static. Multi-response chains are registered dynamically by +the runner at scenario arm time through `POST /__admin/mappings`, then deleted before the next +scenario. This keeps the fixture set small while preserving the same Stellar RPC response bodies. + +Prerequisites: + +- Docker with Compose support. +- Rust/Cargo for `cargo build --bin openzeppelin-relayer`. +- `curl` and `jq`. +- Network access to Stellar testnet, Horizon testnet, and friendbot. +- The local signer keystore is generated automatically at + `testing/wiremock/e2e/local-signer.keystore` if missing, using + `KEYSTORE_PASSPHRASE`, and the generated testnet account is funded through friendbot before + the relayer starts. + +The runner starts WireMock, starts a disposable Redis container for the Redis queue backend, builds +and starts `target/debug/openzeppelin-relayer`, funds the configured Stellar testnet account if +needed, runs the selected scenario set, writes artifacts, and tears the stack down. + +`if-overcap` runs last in the full matrix because it intentionally never lands on-chain and can +leave the relayer-side local sequence cache ahead of chain state. `tal-window-closes` sends an +explicit 10-minute `valid_until` because it waits for status-checker handoff after the fast retry +window closes. + +Artifacts default to `/tmp/oz-relayer-fast-resubmit` and can be overridden with `-o` or +`ARTIFACT_DIR`. The runner writes: + +- `verdicts.json`: JSON Lines, one verdict per scenario. +- `relayer.log`: relayer stdout/stderr. +- `harness.log`: stack setup and harness diagnostics. +- `report.md`: line-count summary, deleted/kept scope, and matrix verdicts. + +Each verdict line has this schema: + +```json +{ + "scenario": "if-once", + "send_calls": 2, + "gaps_s": [5], + "final_status": "confirmed", + "fast_log_seen": true, + "checker_log_seen": false, + "pass": true, + "notes": "" +} +``` + +Gap checks use a tolerance of expected minus 1 second through expected plus 8 seconds. +`tal-window-closes` treats its final checker-rescue gap as `>= 10s`. + ## How It Works WireMock runs as a transparent proxy (`--proxy-all`) to the target RPC endpoint (default: `https://soroban-testnet.stellar.org`). Pre-loaded stub mappings use **scenarios** (state machines) to intercept specific requests: diff --git a/testing/wiremock/e2e/fast-resubmit-config.json b/testing/wiremock/e2e/fast-resubmit-config.json new file mode 100644 index 000000000..d49514ba8 --- /dev/null +++ b/testing/wiremock/e2e/fast-resubmit-config.json @@ -0,0 +1,47 @@ +{ + "relayers": [ + { + "id": "stellar-fast-resubmit", + "name": "Stellar Fast Resubmit E2E", + "network": "testnet", + "paused": false, + "network_type": "stellar", + "signer_id": "local-signer", + "policies": { + "fee_payment_strategy": "relayer", + "min_balance": 0 + } + } + ], + "notifications": [], + "signers": [ + { + "id": "local-signer", + "type": "local", + "config": { + "path": "testing/wiremock/e2e/local-signer.keystore", + "passphrase": { + "type": "env", + "value": "KEYSTORE_PASSPHRASE" + } + } + } + ], + "networks": [ + { + "type": "stellar", + "network": "testnet", + "rpc_urls": [ + "http://localhost:9090" + ], + "explorer_urls": [ + "https://stellar.expert/explorer/testnet" + ], + "average_blocktime_ms": 5000, + "is_testnet": true, + "passphrase": "Test SDF Network ; September 2015", + "horizon_url": "https://horizon-testnet.stellar.org" + } + ], + "plugins": [] +} diff --git a/testing/wiremock/scripts/e2e-fast-resubmit.sh b/testing/wiremock/scripts/e2e-fast-resubmit.sh new file mode 100755 index 000000000..f41b69f8b --- /dev/null +++ b/testing/wiremock/scripts/e2e-fast-resubmit.sh @@ -0,0 +1,659 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +WIREMOCK_DIR="${REPO_ROOT}/testing/wiremock" + +DEFAULT_ARTIFACT_DIR="${DEFAULT_ARTIFACT_DIR:-/tmp/oz-relayer-fast-resubmit}" +ARTIFACT_DIR="${ARTIFACT_DIR:-${DEFAULT_ARTIFACT_DIR}}" +VERDICTS_FILE="${VERDICTS_FILE:-${ARTIFACT_DIR}/verdicts.json}" +REPORT_FILE="${REPORT_FILE:-${ARTIFACT_DIR}/report.md}" +RELAYER_LOG="${RELAYER_LOG:-${ARTIFACT_DIR}/relayer.log}" +HARNESS_LOG="${HARNESS_LOG:-${ARTIFACT_DIR}/harness.log}" + +WIREMOCK_URL="${WIREMOCK_URL:-http://localhost:9090}" +WIREMOCK_PROXY_TARGET="${WIREMOCK_PROXY_TARGET:-https://soroban-testnet.stellar.org}" +RELAYER_ID="${RELAYER_ID:-stellar-fast-resubmit}" +RELAYER_PORT="${RELAYER_PORT:-18080}" +RELAYER_URL="${RELAYER_URL:-http://127.0.0.1:${RELAYER_PORT}}" +RELAYER_CONFIG_DIR="${WIREMOCK_DIR}/e2e" +RELAYER_CONFIG_FILE_NAME="fast-resubmit-config.json" +LOCAL_SIGNER_KEYSTORE="${REPO_ROOT}/testing/wiremock/e2e/local-signer.keystore" + +API_KEY="${API_KEY:-multi-threaded-stellar-example-api-key}" +KEYSTORE_PASSPHRASE="${KEYSTORE_PASSPHRASE:-MtStellarRuntime123!}" +WEBHOOK_SIGNING_KEY="${WEBHOOK_SIGNING_KEY:-example-webhook-signing-key}" +RUN_ID="fast-resubmit-$(date +%s)-$$" +REDIS_CONTAINER="${REDIS_CONTAINER:-oz-relayer-e2e-redis-${RUN_ID}}" +REDIS_URL_FOR_RELAYER="" +RELAYER_PID="" +REQUESTED_SCENARIO="all" +PASS_COUNT=0 +FAIL_COUNT=0 +DYNAMIC_STUB_IDS=() + +RUNNER_LINES_BEFORE=2146 +README_LINES_BEFORE=261 +CHAINED_MAPPING_FILES_BEFORE=9 +CHAINED_MAPPING_LINES_BEFORE=276 +DELETED_CHAINED_MAPPINGS=( + testing/wiremock/mappings/stellar/stellar-insufficient-fee-armed-2.json + testing/wiremock/mappings/stellar/stellar-insufficient-fee-armed-3.json + testing/wiremock/mappings/stellar/stellar-insufficient-fee-armed-4.json + testing/wiremock/mappings/stellar/stellar-insufficient-fee-armed-5.json + testing/wiremock/mappings/stellar/stellar-insufficient-fee-armed-6.json + testing/wiremock/mappings/stellar/stellar-insufficient-fee-armed-7.json + testing/wiremock/mappings/stellar/stellar-try-again-later-armed-2.json + testing/wiremock/mappings/stellar/stellar-try-again-later-armed-3.json + testing/wiremock/mappings/stellar/stellar-try-again-later-armed-4.json +) + +# Keep if-overcap last: it intentionally never lands on-chain and can leave the +# relayer-side sequence cache ahead of chain state. +ALL_SCENARIOS=(if-once if-escalation tal-window tal-window-closes if-overcap) + +set_artifact_dir() { + ARTIFACT_DIR="$1" + VERDICTS_FILE="${ARTIFACT_DIR}/verdicts.json" + REPORT_FILE="${ARTIFACT_DIR}/report.md" + RELAYER_LOG="${ARTIFACT_DIR}/relayer.log" + HARNESS_LOG="${ARTIFACT_DIR}/harness.log" +} + +usage() { + printf 'usage: %s [-o artifact-dir] \n' "$0" + printf 'scenarios: %s\n' "${ALL_SCENARIOS[*]}" +} + +parse_args() { + local requested_set=false + while (($#)); do + case "$1" in + -o|--output-dir) + shift + (($#)) || { usage >&2; exit 64; } + set_artifact_dir "$1" + ;; + -h|--help) + usage + exit 0 + ;; + -*) + usage >&2 + exit 64 + ;; + *) + [[ "${requested_set}" == "false" ]] || { usage >&2; exit 64; } + REQUESTED_SCENARIO="$1" + requested_set=true + ;; + esac + shift + done +} + +log() { + printf '[e2e-fast-resubmit] %s\n' "$*" >&2 + printf '[e2e-fast-resubmit] %s\n' "$*" >>"${HARNESS_LOG}" +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { printf 'missing required command: %s\n' "$1" >&2; exit 127; } +} + +compose_cmd() { + if docker compose version >/dev/null 2>&1; then + printf 'docker compose' + elif command -v docker-compose >/dev/null 2>&1; then + printf 'docker-compose' + else + printf 'missing docker compose\n' >&2 + exit 127 + fi +} + +write_key_address_tool() { + local tool_dir="$1" + mkdir -p "${tool_dir}/src" + cat >"${tool_dir}/Cargo.toml" <<'EOF' +[package] +name = "stellar-keystore-address" +version = "0.1.0" +edition = "2021" + +[dependencies] +ed25519-dalek = { version = "2.2", features = ["pkcs8"] } +oz-keystore = "0.1.4" +stellar-strkey = "0.0.14" +EOF + cat >"${tool_dir}/src/main.rs" <<'EOF' +use std::{env, io, path::PathBuf}; + +fn main() -> Result<(), Box> { + let mut args = env::args().skip(1); + let keystore = args.next().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing keystore path argument") + })?; + let passphrase = args.next().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "missing passphrase argument") + })?; + + let raw_key = oz_keystore::LocalClient::load(PathBuf::from(keystore), passphrase); + let key_bytes: [u8; 32] = raw_key.try_into().map_err(|raw: Vec| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("expected a 32-byte Stellar key, got {} bytes", raw.len()), + ) + })?; + let signing_key = ed25519_dalek::SigningKey::from_bytes(&key_bytes); + let public_key = stellar_strkey::ed25519::PublicKey(signing_key.verifying_key().to_bytes()); + + println!("{public_key}"); + Ok(()) +} +EOF +} + +derive_stellar_address() { + local keystore="$1" tool_dir + tool_dir="${ARTIFACT_DIR}/stellar-keystore-address" + write_key_address_tool "${tool_dir}" + CARGO_TARGET_DIR="${REPO_ROOT}/target" \ + cargo run --quiet --manifest-path "${tool_dir}/Cargo.toml" -- "${keystore}" "${KEYSTORE_PASSPHRASE}" 2>>"${HARNESS_LOG}" +} + +bootstrap_local_signer_keystore() { + local keystore_dir keystore_name address + if [[ -f "${LOCAL_SIGNER_KEYSTORE}" ]]; then + log "local signer keystore exists at ${LOCAL_SIGNER_KEYSTORE}; skipping bootstrap generation" + return 0 + fi + + log "local signer keystore missing; generating ${LOCAL_SIGNER_KEYSTORE}" + keystore_dir="$(dirname "${LOCAL_SIGNER_KEYSTORE}")" + keystore_name="$(basename "${LOCAL_SIGNER_KEYSTORE}")" + ( + cd "${REPO_ROOT}" + cargo run --quiet --example create_key -- \ + --password "${KEYSTORE_PASSPHRASE}" \ + --output-dir "${keystore_dir}" \ + --filename "${keystore_name}" + ) >>"${HARNESS_LOG}" 2>&1 + [[ -f "${LOCAL_SIGNER_KEYSTORE}" ]] || { log "keystore generation did not create ${LOCAL_SIGNER_KEYSTORE}"; return 1; } + + if ! address="$(derive_stellar_address "${LOCAL_SIGNER_KEYSTORE}")"; then + log "failed to derive Stellar address from generated keystore" + return 1 + fi + [[ "${address}" == G* && "${#address}" -eq 56 ]] || { log "failed to derive Stellar address from generated keystore"; return 1; } + ensure_funded "${address}" + log "bootstrapped local signer keystore for ${address}" +} + +delete_dynamic_stubs() { + local id + if ((${#DYNAMIC_STUB_IDS[@]} > 0)); then + for id in "${DYNAMIC_STUB_IDS[@]}"; do + curl -fsS -X DELETE "${WIREMOCK_URL}/__admin/mappings/${id}" >/dev/null 2>&1 || true + done + fi + DYNAMIC_STUB_IDS=() +} + +cleanup() { + local exit_code=$? + delete_dynamic_stubs + if [[ -n "${RELAYER_PID}" ]] && kill -0 "${RELAYER_PID}" >/dev/null 2>&1; then + log "stopping relayer pid ${RELAYER_PID}" + kill "${RELAYER_PID}" >/dev/null 2>&1 || true + wait "${RELAYER_PID}" >/dev/null 2>&1 || true + fi + if docker ps -a --format '{{.Names}}' | grep -qx "${REDIS_CONTAINER}"; then + log "stopping redis container ${REDIS_CONTAINER}" + docker rm -f "${REDIS_CONTAINER}" >/dev/null 2>&1 || true + fi + log "stopping WireMock compose stack" + # shellcheck disable=SC2086 + $(compose_cmd) -f "${WIREMOCK_DIR}/docker-compose.yaml" down >/dev/null 2>&1 || true + exit "${exit_code}" +} + +wait_for_http() { + local url="$1" label="$2" timeout_s="${3:-60}" + local deadline=$(( $(date +%s) + timeout_s )) + until curl -fsS "${url}" >/dev/null 2>&1; do + (( $(date +%s) < deadline )) || { log "timed out waiting for ${label} at ${url}"; return 1; } + sleep 1 + done +} + +start_wiremock() { + log "starting WireMock on ${WIREMOCK_URL}" + # shellcheck disable=SC2086 + WIREMOCK_PORT=9090 WIREMOCK_PROXY_TARGET="${WIREMOCK_PROXY_TARGET}" \ + $(compose_cmd) -f "${WIREMOCK_DIR}/docker-compose.yaml" up -d wiremock >>"${HARNESS_LOG}" 2>&1 + wait_for_http "${WIREMOCK_URL}/__admin/health" "WireMock" 90 +} + +start_redis() { + local deadline redis_port + log "starting disposable Redis container ${REDIS_CONTAINER}" + docker rm -f "${REDIS_CONTAINER}" >/dev/null 2>&1 || true + docker run -d --name "${REDIS_CONTAINER}" -p 127.0.0.1::6379 redis:bookworm >>"${HARNESS_LOG}" 2>&1 + deadline=$(( $(date +%s) + 60 )) + until docker exec "${REDIS_CONTAINER}" redis-cli ping 2>/dev/null | grep -q PONG; do + (( $(date +%s) < deadline )) || { log "timed out waiting for Redis"; return 1; } + sleep 1 + done + redis_port="$(docker port "${REDIS_CONTAINER}" 6379/tcp | sed -E 's/.*:([0-9]+)$/\1/')" + REDIS_URL_FOR_RELAYER="redis://127.0.0.1:${redis_port}" + log "Redis is ready at ${REDIS_URL_FOR_RELAYER}" +} + +start_relayer() { + log "building relayer binary" + ( cd "${REPO_ROOT}" && cargo build --bin openzeppelin-relayer ) >>"${RELAYER_LOG}" 2>&1 + + log "starting relayer at ${RELAYER_URL}; logs: ${RELAYER_LOG}" + ( + cd "${REPO_ROOT}" + env \ + API_KEY="${API_KEY}" \ + KEYSTORE_PASSPHRASE="${KEYSTORE_PASSPHRASE}" \ + WEBHOOK_SIGNING_KEY="${WEBHOOK_SIGNING_KEY}" \ + HOST=127.0.0.1 \ + APP_PORT="${RELAYER_PORT}" \ + CONFIG_DIR="${RELAYER_CONFIG_DIR}" \ + CONFIG_FILE_NAME="${RELAYER_CONFIG_FILE_NAME}" \ + REDIS_URL="${REDIS_URL_FOR_RELAYER}" \ + QUEUE_BACKEND=redis \ + REPOSITORY_STORAGE_TYPE=in_memory \ + REDIS_KEY_PREFIX="${RUN_ID}" \ + METRICS_ENABLED=false \ + ENABLE_SWAGGER=false \ + LOG_LEVEL=debug \ + LOG_FORMAT=compact \ + RPC_BLOCK_PRIVATE_IPS=false \ + RPC_ALLOWED_HOSTS=localhost,127.0.0.1 \ + TOKIO_WORKER_THREADS=4 \ + ACTIX_WORKERS=2 \ + "${REPO_ROOT}/target/debug/openzeppelin-relayer" + ) >>"${RELAYER_LOG}" 2>&1 & + RELAYER_PID=$! + printf '%s\n' "${RELAYER_PID}" >"${ARTIFACT_DIR}/relayer.pid" + wait_for_http "${RELAYER_URL}/api/v1/health" "relayer" 120 +} + +api_get() { + curl -sS --connect-timeout 3 --max-time 15 -H "Authorization: Bearer ${API_KEY}" "${RELAYER_URL}$1" +} + +api_post_json() { + local tmp http body + tmp="$(mktemp)" + http="$(curl -sS --connect-timeout 3 --max-time 15 -o "${tmp}" -w '%{http_code}' \ + -H "Authorization: Bearer ${API_KEY}" -H "Content-Type: application/json" \ + -X POST "${RELAYER_URL}$1" -d "$2" || true)" + body="$(<"${tmp}")" + rm -f "${tmp}" + [[ "${http}" =~ ^2 ]] || { printf '%s' "${body}"; return 1; } + printf '%s' "${body}" +} + +discover_relayer_address() { + local body address + body="$(api_get "/api/v1/relayers/${RELAYER_ID}")" + address="$(jq -r '.data.address // empty' <<<"${body}")" + [[ -n "${address}" ]] || { log "failed to discover relayer address from response: ${body}"; return 1; } + printf '%s' "${address}" +} + +ensure_funded() { + local address="$1" attempt + if curl -fsS "https://horizon-testnet.stellar.org/accounts/${address}" >/dev/null 2>&1; then + log "relayer account already funded: ${address}" + return 0 + fi + for attempt in 1 2 3 4 5; do + log "funding relayer account with friendbot: ${address} (attempt ${attempt}/5)" + curl -fsS "https://friendbot.stellar.org/?addr=${address}" >>"${HARNESS_LOG}" 2>&1 || true + sleep $((attempt * 2)) + curl -fsS "https://horizon-testnet.stellar.org/accounts/${address}" >/dev/null 2>&1 && return 0 + done + log "failed to fund relayer account after retries: ${address}" + return 1 +} + +reset_wiremock() { + delete_dynamic_stubs + curl -fsS -X POST "${WIREMOCK_URL}/__admin/scenarios/reset" >/dev/null + curl -fsS -X DELETE "${WIREMOCK_URL}/__admin/requests" >/dev/null +} + +scenario_meta() { + case "$1" in + if-once) printf 'insufficient-fee\tstellar-insufficient-fee\t1\t2\t5\t120\n' ;; + if-escalation) printf 'insufficient-fee\tstellar-insufficient-fee\t3\t4\t5 10 15\t120\n' ;; + if-overcap) printf 'insufficient-fee\tstellar-insufficient-fee\t7\t7\t5 10 15 20 25 30\t120\n' ;; + tal-window) printf 'try-again-later\tstellar-try-again-later\t3\t4\t5 10 15\t120\n' ;; + tal-window-closes) printf 'try-again-later\tstellar-try-again-later\t4\t5\t5 10 15 10\t300\n' ;; + *) return 1 ;; + esac +} + +arm_state_for_count() { + if (( $1 <= 1 )); then printf 'armed'; else printf 'armed-%s' "$1"; fi +} + +fast_log_for_kind() { + case "$1" in + insufficient-fee) printf 'enqueueing fast resubmit after insufficient fee' ;; + try-again-later) printf 'enqueueing fast resubmit after TRY_AGAIN_LATER' ;; + esac +} + +register_chained_stubs() { + local kind="$1" rejection_count="$2" base_file scenario display_name description + local i next_state mapping response id + (( rejection_count > 1 )) || return 0 + case "${kind}" in + insufficient-fee) + base_file="${WIREMOCK_DIR}/mappings/stellar/stellar-insufficient-fee.json" + scenario="stellar-insufficient-fee" + display_name="ERROR (insufficient fee)" + description="Returns ERROR with tx_insufficient_fee XDR, then decrements to the existing one-shot armed state." + ;; + try-again-later) + base_file="${WIREMOCK_DIR}/mappings/stellar/stellar-try-again-later.json" + scenario="stellar-try-again-later" + display_name="TRY_AGAIN_LATER" + description="Returns TRY_AGAIN_LATER, then decrements to the existing one-shot armed state." + ;; + esac + for ((i = 2; i <= rejection_count; i++)); do + next_state="armed" + (( i == 2 )) || next_state="armed-$((i - 1))" + mapping="$(jq -c --arg name "Stellar sendTransaction -> ${display_name} ${i} of ${i}" \ + --arg required "armed-${i}" --arg next "${next_state}" --arg description "${description}" \ + '.name=$name | .requiredScenarioState=$required | .newScenarioState=$next | .metadata.description=$description' \ + "${base_file}")" + response="$(curl -fsS -X POST "${WIREMOCK_URL}/__admin/mappings" -H 'Content-Type: application/json' -d "${mapping}")" + id="$(jq -r '.id // empty' <<<"${response}")" + [[ -n "${id}" ]] || { log "WireMock did not return an id for ${scenario} armed-${i}"; return 1; } + DYNAMIC_STUB_IDS+=("${id}") + done +} + +arm_scenario() { + curl -fsS -X PUT "${WIREMOCK_URL}/__admin/scenarios/$1/state" \ + -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg state "$2" '{state:$state}')" >/dev/null +} + +send_stats_json() { + curl -fsS "${WIREMOCK_URL}/__admin/requests" | jq -c --arg memo "${1:-}" ' + def body_json: (.request.bodyAsString // .request.body // "{}") as $b | if ($b|type) == "object" then $b else ($b|try fromjson catch {}) end; + def tx_xdr: body_json | ((try .params.transaction catch null) // (try .params[0].transaction catch null) // ""); + def to_epoch: if type == "number" then (if . > 100000000000 then ./1000 else . end) else (tostring | sub("\\+0000$";"Z") | sub("\\.[0-9]+Z$";"Z") | fromdateiso8601) end; + [.requests[] + | select(.request.method == "POST") + | select((body_json | .method // "") == "sendTransaction") + | select($memo == "" or ((try (tx_xdr | @base64d) catch "") | contains($memo))) + | (.request.loggedDate // .loggedDate // .loggedDateString) | to_epoch + ] | sort as $t | {send_calls: ($t|length), gaps_s: [range(1; $t|length) as $i | (((($t[$i] - $t[$i-1]) * 10) | round) / 10)]}' +} + +submit_fresh_transaction() { + local destination="$1" scenario="$2" memo valid_until payload body tx_id + memo="fr-$(date +%s)-${RANDOM}" + valid_until="" + [[ "${scenario}" == "tal-window-closes" ]] && valid_until="$(( $(date +%s) + 600 ))" + payload="$(jq -nc --arg destination "${destination}" --arg memo "${memo}" --arg valid_until "${valid_until}" \ + '{network:"testnet",operations:[{type:"payment",destination:$destination,amount:"1",asset:{type:"native"}}],memo:{type:"text",value:$memo}} + + if $valid_until == "" then {} else {valid_until:$valid_until} end')" + body="$(api_post_json "/api/v1/relayers/${RELAYER_ID}/transactions" "${payload}")" || { printf '%s' "${body}"; return 1; } + tx_id="$(jq -r '.data.id // empty' <<<"${body}")" + [[ -n "${tx_id}" ]] || { log "${scenario}: missing tx id in response: ${body}"; return 1; } + jq -nc --arg tx_id "${tx_id}" --arg memo "${memo}" '{tx_id:$tx_id,memo:$memo}' +} + +poll_transaction() { + local tx_id="$1" scenario="$2" timeout_s="$3" body status reason + local deadline=$(( $(date +%s) + timeout_s )) + status="" + reason="" + while (( $(date +%s) < deadline )); do + body="$(api_get "/api/v1/relayers/${RELAYER_ID}/transactions/${tx_id}")" || true + status="$(jq -r '.data.status // empty' <<<"${body}" 2>/dev/null || true)" + reason="$(jq -r '.data.status_reason // ""' <<<"${body}" 2>/dev/null || true)" + case "${scenario}:${status}" in + if-overcap:failed|*:confirmed|*:mined|*:expired|*:canceled|if-once:submitted) + jq -nc --arg status "${status}" --arg reason "${reason}" '{status:$status,reason:$reason}' + return 0 + ;; + esac + sleep 2 + done + jq -nc --arg status "${status:-timeout}" --arg reason "${reason}" '{status:$status,reason:$reason}' +} + +log_bool_seen() { + awk -v tx_id="$1" -v needle="$2" 'index($0, tx_id) && index($0, needle) { found=1; exit } END { exit found ? 0 : 1 }' "${RELAYER_LOG}" \ + && printf 'true' || printf 'false' +} + +gap_within() { + awk -v actual="$1" -v expected="$2" -v idx="$3" -v scenario="$4" \ + 'BEGIN { if (scenario == "tal-window-closes" && idx == 3) ok = actual >= 10; else ok = actual >= expected - 1 && actual <= expected + 8; exit ok ? 0 : 1 }' +} + +append_note() { + if [[ -n "${notes}" ]]; then notes="${notes}; $1"; else notes="$1"; fi +} + +evaluate_pass() { + local scenario="$1" stats="$2" final_status="$3" status_reason="$4" fast_seen="$5" checker_seen="$6" + local expected_calls="$7" expected_gaps="$8" send_calls gaps_csv notes pass i min_count + local gaps=() expected=() + notes="" + send_calls="$(jq -r '.send_calls' <<<"${stats}")" + gaps_csv="$(jq -r '.gaps_s | join(",")' <<<"${stats}")" + [[ -z "${gaps_csv}" ]] || IFS=',' read -r -a gaps <<<"${gaps_csv}" + expected=(${expected_gaps}) + + [[ "${send_calls}" == "${expected_calls}" ]] || append_note "send_calls expected ${expected_calls} got ${send_calls}" + [[ "${#gaps[@]}" == "${#expected[@]}" ]] || append_note "gaps expected length ${#expected[@]} got ${#gaps[@]}" + min_count="${#gaps[@]}" + (( ${#expected[@]} < min_count )) && min_count="${#expected[@]}" + for ((i = 0; i < min_count; i++)); do + gap_within "${gaps[$i]}" "${expected[$i]}" "${i}" "${scenario}" || append_note "gap[${i}] expected ${expected[$i]}s got ${gaps[$i]}s" + done + + [[ "${fast_seen}" == "true" ]] || append_note "fast log not seen" + if [[ "${scenario}" == "tal-window-closes" ]]; then + [[ "${checker_seen}" == "true" ]] || append_note "checker log not seen" + else + [[ "${checker_seen}" == "false" ]] || append_note "checker log unexpectedly seen" + fi + + case "${scenario}" in + if-overcap) + [[ "${final_status}" == "failed" ]] || append_note "final_status expected failed got ${final_status}" + [[ "${status_reason}" == *"insufficient fee retry limit exceeded (6)"* ]] || append_note "status_reason missing retry cap" + ;; + if-once) + [[ "${final_status}" == "confirmed" || "${final_status}" == "submitted" || "${final_status}" == "mined" ]] || append_note "final_status expected confirmed/submitted got ${final_status}" + ;; + *) + [[ "${final_status}" == "confirmed" || "${final_status}" == "mined" ]] || append_note "final_status expected confirmed got ${final_status}" + ;; + esac + + [[ -z "${notes}" ]] && pass=true || pass=false + jq -nc --argjson pass "${pass}" --arg notes "${notes}" '{pass:$pass,notes:$notes}' +} + +emit_verdict() { + local scenario="$1" stats="$2" final_status="$3" fast_seen="$4" checker_seen="$5" pass="$6" notes="$7" line + line="$(jq -nc --arg scenario "${scenario}" --argjson send_calls "$(jq '.send_calls' <<<"${stats}")" \ + --argjson gaps_s "$(jq -c '.gaps_s' <<<"${stats}")" --arg final_status "${final_status}" \ + --argjson fast_log_seen "${fast_seen}" --argjson checker_log_seen "${checker_seen}" \ + --argjson pass "${pass}" --arg notes "${notes}" \ + '{scenario:$scenario,send_calls:$send_calls,gaps_s:$gaps_s,final_status:$final_status,fast_log_seen:$fast_log_seen,checker_log_seen:$checker_log_seen,pass:$pass,notes:$notes}')" + printf '%s\n' "${line}" + printf '%s\n' "${line}" >>"${VERDICTS_FILE}" +} + +run_scenario() { + local scenario="$1" address="$2" kind rpc_scenario rejection_count expected_calls expected_gaps timeout_s + local arm_state submitted tx_id memo poll stats final_status status_reason fast_seen checker_seen eval pass notes + IFS=$'\t' read -r kind rpc_scenario rejection_count expected_calls expected_gaps timeout_s < <(scenario_meta "${scenario}") + + arm_state="$(arm_state_for_count "${rejection_count}")" + log "running ${scenario}: arm ${rpc_scenario} -> ${arm_state}" + reset_wiremock + register_chained_stubs "${kind}" "${rejection_count}" + arm_scenario "${rpc_scenario}" "${arm_state}" + + if ! submitted="$(submit_fresh_transaction "${address}" "${scenario}")"; then + stats='{"send_calls":0,"gaps_s":[]}' + emit_verdict "${scenario}" "${stats}" "submit_failed" false false false "transaction submit API failed: ${submitted}" + delete_dynamic_stubs + FAIL_COUNT=$((FAIL_COUNT + 1)) + return 0 + fi + + tx_id="$(jq -r '.tx_id // empty' <<<"${submitted}")" + memo="$(jq -r '.memo // empty' <<<"${submitted}")" + if [[ -z "${tx_id}" || -z "${memo}" ]]; then + stats='{"send_calls":0,"gaps_s":[]}' + emit_verdict "${scenario}" "${stats}" "submit_failed" false false false "transaction submit API returned malformed body: ${submitted}" + delete_dynamic_stubs + FAIL_COUNT=$((FAIL_COUNT + 1)) + return 0 + fi + log "${scenario}: tx_id=${tx_id} memo=${memo}" + + poll="$(poll_transaction "${tx_id}" "${scenario}" "${timeout_s}")" + final_status="$(jq -r '.status' <<<"${poll}")" + status_reason="$(jq -r '.reason' <<<"${poll}")" + stats="$(send_stats_json "${memo}")" + fast_seen="$(log_bool_seen "${tx_id}" "$(fast_log_for_kind "${kind}")")" + checker_seen="$(log_bool_seen "${tx_id}" "re-enqueueing submit job for stuck Sent transaction")" + eval="$(evaluate_pass "${scenario}" "${stats}" "${final_status}" "${status_reason}" "${fast_seen}" "${checker_seen}" "${expected_calls}" "${expected_gaps}")" + pass="$(jq -r '.pass' <<<"${eval}")" + notes="$(jq -r '.notes' <<<"${eval}")" + emit_verdict "${scenario}" "${stats}" "${final_status}" "${fast_seen}" "${checker_seen}" "${pass}" "${notes}" + delete_dynamic_stubs + + if [[ "${pass}" == "true" ]]; then + PASS_COUNT=$((PASS_COUNT + 1)) + else + FAIL_COUNT=$((FAIL_COUNT + 1)) + grep -F "${tx_id}" "${RELAYER_LOG}" >"${ARTIFACT_DIR}/${scenario}-${tx_id}.log" || true + log "${scenario}: failed; tx-scoped log excerpt: ${ARTIFACT_DIR}/${scenario}-${tx_id}.log" + fi +} + +select_scenarios() { + local requested="$1" scenario + if [[ "${requested}" == "all" ]]; then + printf '%s\n' "${ALL_SCENARIOS[@]}" + return 0 + fi + for scenario in "${ALL_SCENARIOS[@]}"; do + [[ "${scenario}" != "${requested}" ]] || { printf '%s\n' "${scenario}"; return 0; } + done + usage >&2 + exit 64 +} + +count_chained_mapping_lines() { + local file total=0 + while IFS= read -r file; do + total=$((total + $(wc -l <"${file}" | tr -d ' '))) + done < <(find "${WIREMOCK_DIR}/mappings/stellar" \( -name 'stellar-insufficient-fee-armed-*.json' -o -name 'stellar-try-again-later-armed-*.json' \)) + printf '%s' "${total}" +} + +write_report() { + local runner_lines_after readme_lines_after chained_files_after chained_lines_after passed total mapping + runner_lines_after="$(wc -l <"${BASH_SOURCE[0]}" | tr -d ' ')" + readme_lines_after="$(wc -l <"${WIREMOCK_DIR}/README.md" | tr -d ' ')" + chained_files_after="$(find "${WIREMOCK_DIR}/mappings/stellar" \( -name 'stellar-insufficient-fee-armed-*.json' -o -name 'stellar-try-again-later-armed-*.json' \) | wc -l | tr -d ' ')" + chained_lines_after="$(count_chained_mapping_lines)" + passed="$(jq -s 'map(select(.pass == true)) | length' "${VERDICTS_FILE}")" + total="$(jq -s 'length' "${VERDICTS_FILE}")" + + { + printf '# Stellar fast-resubmit E2E report\n\n' + printf -- '- Generated at: `%s`\n' "$(date -u +%FT%TZ)" + printf -- '- Artifact directory: `%s`\n' "${ARTIFACT_DIR}" + printf -- '- Verdicts: `%s`\n' "${VERDICTS_FILE}" + printf -- '- Relayer log: `%s`\n' "${RELAYER_LOG}" + printf -- '- Harness log: `%s`\n\n' "${HARNESS_LOG}" + printf '## Slimming summary\n\n' + printf '| Item | Before | After |\n|---|---:|---:|\n' + printf '| Runner lines | %s | %s |\n' "${RUNNER_LINES_BEFORE}" "${runner_lines_after}" + printf '| README lines | %s | %s |\n' "${README_LINES_BEFORE}" "${readme_lines_after}" + printf '| Static chained mapping files | %s | %s |\n' "${CHAINED_MAPPING_FILES_BEFORE}" "${chained_files_after}" + printf '| Static chained mapping lines | %s | %s |\n\n' "${CHAINED_MAPPING_LINES_BEFORE}" "${chained_lines_after}" + printf 'Deleted static chained mappings; equivalent scenario-state stubs are now registered dynamically and removed between scenarios:\n\n' + for mapping in "${DELETED_CHAINED_MAPPINGS[@]}"; do printf -- '- `%s`\n' "${mapping}"; done + printf '\nKept: WireMock proxy, disposable Redis queue backend, Cargo-built relayer, Stellar testnet account funding, five scenario checks, JSONL verdicts, report, and teardown.\n' + printf '\nDeleted: channels-parity profile, LocalStack/SQS setup, env generation, ledger/substitution reporting, closed-loop soak mode, multi-relayer generation, and residual scoring.\n\n' + printf '## Matrix verdicts\n\n' + printf 'Passed `%s/%s` scenarios.\n\n' "${passed}" "${total}" + printf '```jsonl\n' + cat "${VERDICTS_FILE}" + printf '```\n\n' + printf 'Each verdict has `scenario`, `send_calls`, `gaps_s`, `final_status`, `fast_log_seen`, `checker_log_seen`, `pass`, and `notes`.\n' + } >"${REPORT_FILE}" +} + +main() { + local scenarios=() scenario address + parse_args "$@" + mkdir -p "${ARTIFACT_DIR}" + : >"${HARNESS_LOG}" + : >"${RELAYER_LOG}" + : >"${VERDICTS_FILE}" + + need_cmd docker + need_cmd curl + need_cmd jq + need_cmd cargo + + local selection + if ! selection="$(select_scenarios "${REQUESTED_SCENARIO}")"; then + exit 64 + fi + scenarios=() + while IFS= read -r scenario; do + [[ -n "${scenario}" ]] && scenarios+=("${scenario}") + done <<< "${selection}" + if (( ${#scenarios[@]} == 0 )); then + log "error: no scenarios selected for '${REQUESTED_SCENARIO}'" >&2 + exit 64 + fi + + trap cleanup EXIT + + bootstrap_local_signer_keystore + + start_wiremock + start_redis + start_relayer + + address="$(discover_relayer_address)" + ensure_funded "${address}" + log "using relayer address ${address}" + for scenario in "${scenarios[@]}"; do run_scenario "${scenario}" "${address}"; done + + write_report + log "pass=${PASS_COUNT} fail=${FAIL_COUNT}; verdicts=${VERDICTS_FILE}; report=${REPORT_FILE}" + (( FAIL_COUNT == 0 )) +} + +main "$@"