diff --git a/blockchain-custody/blockchain-custody/build.gradle.kts b/blockchain-custody/blockchain-custody/build.gradle.kts index 222d60cc..4001ce44 100644 --- a/blockchain-custody/blockchain-custody/build.gradle.kts +++ b/blockchain-custody/blockchain-custody/build.gradle.kts @@ -100,6 +100,7 @@ dependencies { // Resilience4j implementation("io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion") implementation("io.github.resilience4j:resilience4j-circuitbreaker:$resilience4jVersion") + implementation("io.github.resilience4j:resilience4j-retry:$resilience4jVersion") // Dev custody adapter — EVM transaction signing implementation("org.web3j:core:$web3jVersion") { diff --git a/blockchain-custody/blockchain-custody/src/integration-test/resources/application-integration-test.yml b/blockchain-custody/blockchain-custody/src/integration-test/resources/application-integration-test.yml index dc5a6b67..b4b61cfe 100644 --- a/blockchain-custody/blockchain-custody/src/integration-test/resources/application-integration-test.yml +++ b/blockchain-custody/blockchain-custody/src/integration-test/resources/application-integration-test.yml @@ -34,6 +34,16 @@ app: fallback-adapters: enabled: true +resilience4j: + retry: + instances: + fireblocks: + wait-duration: 10ms + evmRpc: + wait-duration: 10ms + solanaRpc: + wait-duration: 10ms + logging: level: com.stablecoin.payments: DEBUG diff --git a/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapter.java b/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapter.java index 19c7984e..cd8a4a39 100644 --- a/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapter.java +++ b/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapter.java @@ -4,6 +4,7 @@ import com.stablecoin.payments.custody.domain.port.ChainRpcProvider; import com.stablecoin.payments.custody.domain.port.TransactionReceipt; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -61,6 +62,7 @@ public EvmRpcAdapter(EvmChainProperties properties) { } @Override + @Retry(name = "evmRpc", fallbackMethod = "getTransactionReceiptFallback") @CircuitBreaker(name = "evmRpc", fallbackMethod = "getTransactionReceiptFallback") public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash) { log.info("[EVM-RPC] Getting transaction receipt chain={} txHash={}", chainId.value(), txHash); @@ -88,6 +90,7 @@ public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash) } @Override + @Retry(name = "evmRpc", fallbackMethod = "getLatestBlockNumberFallback") @CircuitBreaker(name = "evmRpc", fallbackMethod = "getLatestBlockNumberFallback") public long getLatestBlockNumber(ChainId chainId) { log.info("[EVM-RPC] Getting latest block number chain={}", chainId.value()); @@ -100,6 +103,7 @@ public long getLatestBlockNumber(ChainId chainId) { } @Override + @Retry(name = "evmRpc", fallbackMethod = "getTokenBalanceFallback") @CircuitBreaker(name = "evmRpc", fallbackMethod = "getTokenBalanceFallback") public BigDecimal getTokenBalance(ChainId chainId, String address, String tokenContract) { log.info("[EVM-RPC] Getting token balance chain={} address={} contract={}", @@ -177,23 +181,23 @@ static String encodeBalanceOfCall(String address) { @SuppressWarnings("unused") private TransactionReceipt getTransactionReceiptFallback(ChainId chainId, String txHash, Exception ex) { - log.error("[EVM-RPC] Circuit breaker open — getTransactionReceipt failed chain={} txHash={}", - chainId.value(), txHash, ex); + log.error("[EVM-RPC] Resilience fallback — getTransactionReceipt failed chain={} txHash={} due to {}", + chainId.value(), txHash, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("EVM RPC unavailable for getTransactionReceipt", ex); } @SuppressWarnings("unused") private long getLatestBlockNumberFallback(ChainId chainId, Exception ex) { - log.error("[EVM-RPC] Circuit breaker open — getLatestBlockNumber failed chain={}", - chainId.value(), ex); + log.error("[EVM-RPC] Resilience fallback — getLatestBlockNumber failed chain={} due to {}", + chainId.value(), ex.getClass().getSimpleName(), ex); throw new IllegalStateException("EVM RPC unavailable for getLatestBlockNumber", ex); } @SuppressWarnings("unused") private BigDecimal getTokenBalanceFallback(ChainId chainId, String address, String tokenContract, Exception ex) { - log.error("[EVM-RPC] Circuit breaker open — getTokenBalance failed chain={} address={}", - chainId.value(), address, ex); + log.error("[EVM-RPC] Resilience fallback — getTokenBalance failed chain={} address={} due to {}", + chainId.value(), address, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("EVM RPC unavailable for getTokenBalance", ex); } diff --git a/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapter.java b/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapter.java index 1481daff..b2e6b90e 100644 --- a/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapter.java +++ b/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapter.java @@ -7,6 +7,7 @@ import com.stablecoin.payments.custody.domain.port.SignResult; import com.stablecoin.payments.custody.domain.port.TransactionStatus; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -68,6 +69,7 @@ public FireblocksCustodyAdapter(FireblocksProperties properties) { } @Override + @Retry(name = "fireblocks", fallbackMethod = "signAndSubmitFallback") @CircuitBreaker(name = "fireblocks", fallbackMethod = "signAndSubmitFallback") public SignResult signAndSubmit(SignRequest request) { log.info("[FIREBLOCKS] Signing and submitting transfer transferId={} chain={} to={}", @@ -109,6 +111,7 @@ public SignResult signAndSubmit(SignRequest request) { } @Override + @Retry(name = "fireblocks", fallbackMethod = "getTransactionStatusFallback") @CircuitBreaker(name = "fireblocks", fallbackMethod = "getTransactionStatusFallback") public TransactionStatus getTransactionStatus(String txId) { log.info("[FIREBLOCKS] Getting transaction status txId={}", txId); @@ -131,14 +134,15 @@ public TransactionStatus getTransactionStatus(String txId) { @SuppressWarnings("unused") private SignResult signAndSubmitFallback(SignRequest request, Exception ex) { - log.error("[FIREBLOCKS] Circuit breaker open — signAndSubmit failed transferId={}", - request.transferId(), ex); + log.error("[FIREBLOCKS] Resilience fallback — signAndSubmit failed transferId={} due to {}", + request.transferId(), ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Fireblocks custody unavailable", ex); } @SuppressWarnings("unused") private TransactionStatus getTransactionStatusFallback(String txId, Exception ex) { - log.error("[FIREBLOCKS] Circuit breaker open — getTransactionStatus failed txId={}", txId, ex); + log.error("[FIREBLOCKS] Resilience fallback — getTransactionStatus failed txId={} due to {}", + txId, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Fireblocks custody unavailable", ex); } diff --git a/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapter.java b/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapter.java index 532f2042..d95d8872 100644 --- a/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapter.java +++ b/blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapter.java @@ -4,6 +4,7 @@ import com.stablecoin.payments.custody.domain.port.ChainRpcProvider; import com.stablecoin.payments.custody.domain.port.TransactionReceipt; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -55,6 +56,7 @@ public SolanaRpcAdapter(SolanaChainProperties properties) { } @Override + @Retry(name = "solanaRpc", fallbackMethod = "getTransactionReceiptFallback") @CircuitBreaker(name = "solanaRpc", fallbackMethod = "getTransactionReceiptFallback") public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash) { log.info("[SOLANA-RPC] Getting transaction chain={} signature={}", chainId.value(), txHash); @@ -86,6 +88,7 @@ public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash) } @Override + @Retry(name = "solanaRpc", fallbackMethod = "getLatestBlockNumberFallback") @CircuitBreaker(name = "solanaRpc", fallbackMethod = "getLatestBlockNumberFallback") public long getLatestBlockNumber(ChainId chainId) { log.info("[SOLANA-RPC] Getting current slot chain={}", chainId.value()); @@ -99,6 +102,7 @@ public long getLatestBlockNumber(ChainId chainId) { } @Override + @Retry(name = "solanaRpc", fallbackMethod = "getTokenBalanceFallback") @CircuitBreaker(name = "solanaRpc", fallbackMethod = "getTokenBalanceFallback") public BigDecimal getTokenBalance(ChainId chainId, String address, String tokenContract) { log.info("[SOLANA-RPC] Getting SPL token balance chain={} owner={} mint={}", @@ -167,23 +171,23 @@ JsonNode callJsonRpc(String method, Object... params) { @SuppressWarnings("unused") private TransactionReceipt getTransactionReceiptFallback(ChainId chainId, String txHash, Exception ex) { - log.error("[SOLANA-RPC] Circuit breaker open - getTransactionReceipt failed chain={} signature={}", - chainId.value(), txHash, ex); + log.error("[SOLANA-RPC] Resilience fallback — getTransactionReceipt failed chain={} signature={} due to {}", + chainId.value(), txHash, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Solana RPC unavailable for getTransactionReceipt", ex); } @SuppressWarnings("unused") private long getLatestBlockNumberFallback(ChainId chainId, Exception ex) { - log.error("[SOLANA-RPC] Circuit breaker open - getLatestBlockNumber failed chain={}", - chainId.value(), ex); + log.error("[SOLANA-RPC] Resilience fallback — getLatestBlockNumber failed chain={} due to {}", + chainId.value(), ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Solana RPC unavailable for getSlot", ex); } @SuppressWarnings("unused") private BigDecimal getTokenBalanceFallback(ChainId chainId, String address, String tokenContract, Exception ex) { - log.error("[SOLANA-RPC] Circuit breaker open - getTokenBalance failed chain={} owner={}", - chainId.value(), address, ex); + log.error("[SOLANA-RPC] Resilience fallback — getTokenBalance failed chain={} owner={} due to {}", + chainId.value(), address, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Solana RPC unavailable for getTokenBalance", ex); } diff --git a/blockchain-custody/blockchain-custody/src/main/resources/application.yml b/blockchain-custody/blockchain-custody/src/main/resources/application.yml index 633206db..6bc91ebc 100644 --- a/blockchain-custody/blockchain-custody/src/main/resources/application.yml +++ b/blockchain-custody/blockchain-custody/src/main/resources/application.yml @@ -176,6 +176,55 @@ app: balance: low-alert-threshold-usd: 50000 +resilience4j: + circuitbreaker: + circuit-breaker-aspect-order: 1 + retry: + retry-aspect-order: 3 + instances: + fireblocks: + max-attempts: 3 + wait-duration: 1s + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + evmRpc: + max-attempts: 5 + wait-duration: 500ms + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + solanaRpc: + max-attempts: 5 + wait-duration: 500ms + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + logging: level: com.stablecoin.payments: INFO diff --git a/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapterRetryTest.java b/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapterRetryTest.java new file mode 100644 index 00000000..4a8e5b22 --- /dev/null +++ b/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapterRetryTest.java @@ -0,0 +1,148 @@ +package com.stablecoin.payments.custody.infrastructure.provider.evm; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.custody.domain.model.ChainId; +import com.stablecoin.payments.custody.domain.port.ChainRpcProvider; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.util.Map; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = EvmRpcAdapterRetryTest.TestConfig.class) +@DisplayName("EvmRpcAdapter retry behavior") +class EvmRpcAdapterRetryTest { + + private static WireMockServer wireMock; + private static final ChainId BASE_CHAIN = new ChainId("base"); + + @Autowired + private ChainRpcProvider chainRpcProvider; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.custody.evm.enabled", () -> "true"); + registry.add("resilience4j.retry.instances.evmRpc.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.evmRpc.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.evmRpc.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.evmRpc.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + {"jsonrpc":"2.0","id":1,"result":"0x1a4"} + """))); + + var result = chainRpcProvider.getLatestBlockNumber(BASE_CHAIN); + + assertThat(result).isEqualTo(420L); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/")) + .willReturn(aResponse().withStatus(400))); + + assertThatThrownBy(() -> chainRpcProvider.getLatestBlockNumber(BASE_CHAIN)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("EVM RPC unavailable for getLatestBlockNumber"); + + wireMock.verify(1, postRequestedFor(urlEqualTo("/"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> chainRpcProvider.getLatestBlockNumber(BASE_CHAIN)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("EVM RPC unavailable for getLatestBlockNumber"); + + wireMock.verify(3, postRequestedFor(urlEqualTo("/"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + EvmChainProperties evmChainProperties() { + return new EvmChainProperties(true, Map.of( + "base", new EvmChainProperties.ChainRpcConfig( + wireMock.baseUrl(), 84532L, + "0x036CbD53842c5426634e7929541eC2318f3dCF7e", + 5000, 10000) + )); + } + + @Bean + EvmRpcAdapter evmRpcAdapter(EvmChainProperties properties) { + return new EvmRpcAdapter(properties); + } + } +} diff --git a/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapterRetryTest.java b/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapterRetryTest.java new file mode 100644 index 00000000..0ac96b6a --- /dev/null +++ b/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapterRetryTest.java @@ -0,0 +1,238 @@ +package com.stablecoin.payments.custody.infrastructure.provider.fireblocks; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.custody.domain.port.CustodyEngine; +import com.stablecoin.payments.custody.domain.port.SignResult; +import com.stablecoin.payments.custody.domain.port.TransactionStatus; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.security.KeyPairGenerator; +import java.util.Base64; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static com.stablecoin.payments.custody.fixtures.CustodyEngineFixtures.aSignRequest; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = FireblocksCustodyAdapterRetryTest.TestConfig.class) +@DisplayName("FireblocksCustodyAdapter retry behavior") +class FireblocksCustodyAdapterRetryTest { + + private static WireMockServer wireMock; + private static String testPrivateKeyPem; + + @Autowired + private CustodyEngine custodyEngine; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() throws Exception { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + + var keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(2048); + var keyPair = keyGen.generateKeyPair(); + var privateKeyBytes = keyPair.getPrivate().getEncoded(); + var base64Key = Base64.getEncoder().encodeToString(privateKeyBytes); + testPrivateKeyPem = "-----BEGIN PRIVATE KEY-----\n" + base64Key + "\n-----END PRIVATE KEY-----"; + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.custody.provider", () -> "fireblocks"); + registry.add("resilience4j.retry.instances.fireblocks.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.fireblocks.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.fireblocks.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.fireblocks.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/v1/transactions")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/v1/transactions")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "id": "fb-tx-001", + "status": "SUBMITTED" + } + """))); + + var result = custodyEngine.signAndSubmit(aSignRequest()); + + var expected = new SignResult(null, "fb-tx-001"); + assertThat(result).usingRecursiveComparison().isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/v1/transactions")) + .willReturn(aResponse() + .withStatus(400) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "message": "Invalid asset ID", + "code": 1000 + } + """))); + + assertThatThrownBy(() -> custodyEngine.signAndSubmit(aSignRequest())) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Fireblocks custody unavailable"); + + wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/transactions"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/v1/transactions")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> custodyEngine.signAndSubmit(aSignRequest())) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Fireblocks custody unavailable"); + + wireMock.verify(3, postRequestedFor(urlEqualTo("/v1/transactions"))); + } + + @Test + @DisplayName("should retry getTransactionStatus on transient 503 then succeed") + void shouldRetryGetTransactionStatusOnTransientFailureThenSucceed() { + wireMock.stubFor(get(urlPathMatching("/v1/transactions/.+")) + .inScenario("status-retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("status-second-attempt")); + + wireMock.stubFor(get(urlPathMatching("/v1/transactions/.+")) + .inScenario("status-retry-then-success") + .whenScenarioStateIs("status-second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "id": "fb-tx-001", + "status": "COMPLETED", + "txHash": "0xabc123", + "numOfConfirmations": 12 + } + """))); + + var result = custodyEngine.getTransactionStatus("fb-tx-001"); + + var expected = new TransactionStatus("COMPLETED", "0xabc123", 12); + assertThat(result).usingRecursiveComparison().isEqualTo(expected); + } + + @Test + @DisplayName("should not retry getTransactionStatus on 400 client error") + void shouldNotRetryGetTransactionStatusOnClientError() { + wireMock.stubFor(get(urlPathMatching("/v1/transactions/.+")) + .willReturn(aResponse() + .withStatus(400) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "message": "Invalid transaction ID", + "code": 1000 + } + """))); + + assertThatThrownBy(() -> custodyEngine.getTransactionStatus("invalid-tx")) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Fireblocks custody unavailable"); + + wireMock.verify(1, getRequestedFor(urlPathMatching("/v1/transactions/.+"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 for getTransactionStatus") + void shouldExhaustRetriesForGetTransactionStatus() { + wireMock.stubFor(get(urlPathMatching("/v1/transactions/.+")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> custodyEngine.getTransactionStatus("fb-tx-001")) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Fireblocks custody unavailable"); + + wireMock.verify(3, getRequestedFor(urlPathMatching("/v1/transactions/.+"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + FireblocksProperties fireblocksProperties() { + return new FireblocksProperties( + wireMock.baseUrl(), + "fb-test-api-key", + testPrivateKeyPem, + "42", + 10 + ); + } + + @Bean + FireblocksCustodyAdapter fireblocksCustodyAdapter(FireblocksProperties properties) { + return new FireblocksCustodyAdapter(properties); + } + } +} diff --git a/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapterRetryTest.java b/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapterRetryTest.java new file mode 100644 index 00000000..f1719619 --- /dev/null +++ b/blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapterRetryTest.java @@ -0,0 +1,148 @@ +package com.stablecoin.payments.custody.infrastructure.provider.solana; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.custody.domain.model.ChainId; +import com.stablecoin.payments.custody.domain.port.ChainRpcProvider; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = SolanaRpcAdapterRetryTest.TestConfig.class) +@DisplayName("SolanaRpcAdapter retry behavior") +class SolanaRpcAdapterRetryTest { + + private static WireMockServer wireMock; + private static final ChainId SOLANA_CHAIN = new ChainId("solana"); + + @Autowired + private ChainRpcProvider chainRpcProvider; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.custody.solana.enabled", () -> "true"); + registry.add("resilience4j.retry.instances.solanaRpc.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.solanaRpc.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.solanaRpc.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.solanaRpc.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + {"jsonrpc":"2.0","id":1,"result":285630456} + """))); + + var result = chainRpcProvider.getLatestBlockNumber(SOLANA_CHAIN); + + assertThat(result).isEqualTo(285630456L); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/")) + .willReturn(aResponse().withStatus(400))); + + assertThatThrownBy(() -> chainRpcProvider.getLatestBlockNumber(SOLANA_CHAIN)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Solana RPC unavailable for getSlot"); + + wireMock.verify(1, postRequestedFor(urlEqualTo("/"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> chainRpcProvider.getLatestBlockNumber(SOLANA_CHAIN)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Solana RPC unavailable for getSlot"); + + wireMock.verify(3, postRequestedFor(urlEqualTo("/"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + SolanaChainProperties solanaChainProperties() { + return new SolanaChainProperties( + true, + wireMock.baseUrl(), + "4zMMC9srt5Ri5X14GAgXhaHii3GnPAEERYPJgZJDncDU", + "confirmed", + 5000, + 10000 + ); + } + + @Bean + SolanaRpcAdapter solanaRpcAdapter(SolanaChainProperties properties) { + return new SolanaRpcAdapter(properties); + } + } +} diff --git a/compliance-travel-rule/compliance-travel-rule/build.gradle.kts b/compliance-travel-rule/compliance-travel-rule/build.gradle.kts index 4cf04efb..2cfb4af1 100644 --- a/compliance-travel-rule/compliance-travel-rule/build.gradle.kts +++ b/compliance-travel-rule/compliance-travel-rule/build.gradle.kts @@ -101,6 +101,7 @@ dependencies { // Resilience4j implementation("io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion") implementation("io.github.resilience4j:resilience4j-circuitbreaker:$resilience4jVersion") + implementation("io.github.resilience4j:resilience4j-retry:$resilience4jVersion") // MapStruct (compiler args set below in JavaCompile task) implementation("org.mapstruct:mapstruct:$mapstructVersion") diff --git a/compliance-travel-rule/compliance-travel-rule/src/integration-test/resources/application-integration-test.yml b/compliance-travel-rule/compliance-travel-rule/src/integration-test/resources/application-integration-test.yml index db54167a..0c8f95ba 100644 --- a/compliance-travel-rule/compliance-travel-rule/src/integration-test/resources/application-integration-test.yml +++ b/compliance-travel-rule/compliance-travel-rule/src/integration-test/resources/application-integration-test.yml @@ -32,6 +32,16 @@ app: fallback-adapters: enabled: true +resilience4j: + retry: + instances: + sanctions: + wait-duration: 10ms + kyc: + wait-duration: 10ms + aml: + wait-duration: 10ms + logging: level: com.stablecoin.payments: DEBUG diff --git a/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapter.java b/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapter.java index 17f41a0e..86239c6e 100644 --- a/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapter.java +++ b/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapter.java @@ -3,6 +3,7 @@ import com.stablecoin.payments.compliance.domain.model.AmlResult; import com.stablecoin.payments.compliance.domain.port.AmlProvider; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -49,6 +50,7 @@ public ChainalysisAmlAdapter(ChainalysisProperties properties) { } @Override + @Retry(name = "aml", fallbackMethod = "analyzeFallback") @CircuitBreaker(name = "aml", fallbackMethod = "analyzeFallback") public AmlResult analyze(UUID senderId, UUID recipientId) { log.info("[CHAINALYSIS] Analyzing sender={} recipient={}", senderId, recipientId); @@ -159,8 +161,8 @@ private String buildProviderRef(UUID senderId, UUID recipientId) { @SuppressWarnings("unused") private AmlResult analyzeFallback(UUID senderId, UUID recipientId, Exception ex) { - log.error("[CHAINALYSIS] Circuit breaker open — AML analysis failed sender={} recipient={}", - senderId, recipientId, ex); + log.error("[CHAINALYSIS] Resilience fallback — AML analysis failed sender={} recipient={} due to {}", + senderId, recipientId, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("AML screening unavailable", ex); } } diff --git a/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapter.java b/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapter.java index e58dda24..7478f102 100644 --- a/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapter.java +++ b/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapter.java @@ -5,6 +5,7 @@ import com.stablecoin.payments.compliance.domain.model.KycTier; import com.stablecoin.payments.compliance.domain.port.KycProvider; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -54,6 +55,7 @@ public OnfidoKycAdapter(OnfidoProperties properties, StringRedisTemplate redisTe } @Override + @Retry(name = "kyc", fallbackMethod = "verifyFallback") @CircuitBreaker(name = "kyc", fallbackMethod = "verifyFallback") public KycResult verify(UUID senderId, UUID recipientId) { log.info("[ONFIDO] KYC verification sender={} recipient={}", senderId, recipientId); @@ -149,8 +151,8 @@ private void putInCache(UUID customerId, CustomerKycResult result) { @SuppressWarnings("unused") private KycResult verifyFallback(UUID senderId, UUID recipientId, Exception ex) { - log.error("[ONFIDO] Circuit breaker open — KYC verification failed sender={} recipient={}", - senderId, recipientId, ex); + log.error("[ONFIDO] Resilience fallback — KYC verification failed sender={} recipient={} due to {}", + senderId, recipientId, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("KYC verification unavailable", ex); } diff --git a/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapter.java b/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapter.java index fa60879c..43a8cf93 100644 --- a/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapter.java +++ b/compliance-travel-rule/compliance-travel-rule/src/main/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapter.java @@ -3,6 +3,7 @@ import com.stablecoin.payments.compliance.domain.model.SanctionsResult; import com.stablecoin.payments.compliance.domain.port.SanctionsProvider; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -45,6 +46,7 @@ public WorldCheckSanctionsAdapter(WorldCheckProperties properties) { } @Override + @Retry(name = "sanctions", fallbackMethod = "screenFallback") @CircuitBreaker(name = "sanctions", fallbackMethod = "screenFallback") public SanctionsResult screen(UUID senderId, UUID recipientId) { log.info("[WORLD-CHECK] Screening sender={} recipient={}", senderId, recipientId); @@ -160,8 +162,8 @@ private String buildProviderRef(WorldCheckScreeningResponse senderResp, @SuppressWarnings("unused") private SanctionsResult screenFallback(UUID senderId, UUID recipientId, Exception ex) { - log.error("[WORLD-CHECK] Circuit breaker open — screening failed sender={} recipient={}", - senderId, recipientId, ex); + log.error("[WORLD-CHECK] Resilience fallback — screening failed sender={} recipient={} due to {}", + senderId, recipientId, ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Sanctions screening unavailable", ex); } } diff --git a/compliance-travel-rule/compliance-travel-rule/src/main/resources/application.yml b/compliance-travel-rule/compliance-travel-rule/src/main/resources/application.yml index 8efd5abb..8cc6c671 100644 --- a/compliance-travel-rule/compliance-travel-rule/src/main/resources/application.yml +++ b/compliance-travel-rule/compliance-travel-rule/src/main/resources/application.yml @@ -129,6 +129,49 @@ app: US-SY: 25 US-CU: 20 +resilience4j: + circuitbreaker: + circuit-breaker-aspect-order: 1 + retry: + retry-aspect-order: 3 + instances: + sanctions: + max-attempts: 3 + wait-duration: 500ms + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + kyc: + max-attempts: 3 + wait-duration: 500ms + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + aml: + max-attempts: 3 + wait-duration: 500ms + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + logging: level: com.stablecoin.payments: INFO diff --git a/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapterRetryTest.java b/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapterRetryTest.java new file mode 100644 index 00000000..7f7ceb3b --- /dev/null +++ b/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/chainalysis/ChainalysisAmlAdapterRetryTest.java @@ -0,0 +1,175 @@ +package com.stablecoin.payments.compliance.infrastructure.provider.chainalysis; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.compliance.domain.model.AmlResult; +import com.stablecoin.payments.compliance.domain.port.AmlProvider; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.util.List; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = ChainalysisAmlAdapterRetryTest.TestConfig.class) +@DisplayName("ChainalysisAmlAdapter retry behavior") +class ChainalysisAmlAdapterRetryTest { + + private static WireMockServer wireMock; + + @Autowired + private AmlProvider amlProvider; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + private static final UUID SENDER_ID = UUID.fromString("11111111-1111-1111-1111-111111111111"); + private static final UUID RECIPIENT_ID = UUID.fromString("22222222-2222-2222-2222-222222222222"); + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.aml.provider", () -> "chainalysis"); + registry.add("resilience4j.retry.instances.aml.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.aml.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.aml.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.aml.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + // Register transfer always succeeds + wireMock.stubFor(post(urlPathMatching("/v2/users/.+/transfers")) + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{\"externalId\":\"transfer-registered\"}"))); + + // First GET fails with 503, second succeeds + wireMock.stubFor(get(urlPathMatching("/v2/users/.+/transfers")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(get(urlPathMatching("/v2/users/.+/transfers")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "updatedAt": "2026-03-08T00:00:00Z", + "asset": "USDC", + "rating": "LOW", + "alerts": [] + } + """))); + + var result = amlProvider.analyze(SENDER_ID, RECIPIENT_ID); + + var expected = AmlResult.builder() + .flagged(false) + .flagReasons(List.of()) + .provider("chainalysis") + .providerRef("chainalysis:%s/%s".formatted(SENDER_ID, RECIPIENT_ID)) + .build(); + assertThat(result) + .usingRecursiveComparison() + .ignoringFields("amlResultId", "checkId", "screenedAt", "chainAnalysis") + .isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlPathMatching("/v2/users/.+/transfers")) + .willReturn(aResponse().withStatus(400))); + + assertThatThrownBy(() -> amlProvider.analyze(SENDER_ID, RECIPIENT_ID)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("AML screening unavailable"); + + // 400 is not retried — exactly 1 POST call + wireMock.verify(1, postRequestedFor(urlPathMatching("/v2/users/.+/transfers"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlPathMatching("/v2/users/.+/transfers")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> amlProvider.analyze(SENDER_ID, RECIPIENT_ID)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("AML screening unavailable"); + + // max-attempts=3 => 3 calls for the first registerTransfer call + wireMock.verify(3, postRequestedFor(urlPathMatching("/v2/users/.+/transfers"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + ChainalysisProperties chainalysisProperties() { + return new ChainalysisProperties( + wireMock.baseUrl() + "/v2", + "test-api-key", + 2 + ); + } + + @Bean + ChainalysisAmlAdapter chainalysisAmlAdapter(ChainalysisProperties properties) { + return new ChainalysisAmlAdapter(properties); + } + } +} diff --git a/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapterRetryTest.java b/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapterRetryTest.java new file mode 100644 index 00000000..0b35b297 --- /dev/null +++ b/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/onfido/OnfidoKycAdapterRetryTest.java @@ -0,0 +1,186 @@ +package com.stablecoin.payments.compliance.infrastructure.provider.onfido; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.compliance.domain.model.KycResult; +import com.stablecoin.payments.compliance.domain.model.KycStatus; +import com.stablecoin.payments.compliance.domain.model.KycTier; +import com.stablecoin.payments.compliance.domain.port.KycProvider; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.ValueOperations; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; + +@SpringBootTest(classes = OnfidoKycAdapterRetryTest.TestConfig.class) +@DisplayName("OnfidoKycAdapter retry behavior") +class OnfidoKycAdapterRetryTest { + + private static WireMockServer wireMock; + private static StringRedisTemplate redisTemplate; + private static ValueOperations valueOps; + + @Autowired + private KycProvider kycProvider; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + private static final UUID SENDER_ID = UUID.fromString("11111111-1111-1111-1111-111111111111"); + private static final UUID RECIPIENT_ID = UUID.fromString("22222222-2222-2222-2222-222222222222"); + + @BeforeAll + @SuppressWarnings("unchecked") + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + redisTemplate = mock(StringRedisTemplate.class); + valueOps = mock(ValueOperations.class); + given(redisTemplate.opsForValue()).willReturn(valueOps); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.kyc.provider", () -> "onfido"); + registry.add("resilience4j.retry.instances.kyc.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.kyc.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.kyc.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.kyc.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + given(valueOps.get("kyc:status:" + SENDER_ID)).willReturn(null); + given(valueOps.get("kyc:status:" + RECIPIENT_ID)).willReturn(null); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(get(urlPathEqualTo("/v3.6/checks")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(get(urlPathEqualTo("/v3.6/checks")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "checks": [{ + "id": "chk-001", + "status": "complete", + "result": "clear", + "applicant_id": "app-001", + "report_ids": ["rpt-001"] + }] + } + """))); + + var result = kycProvider.verify(SENDER_ID, RECIPIENT_ID); + + var expected = KycResult.builder() + .senderKycTier(KycTier.KYC_TIER_2) + .senderStatus(KycStatus.VERIFIED) + .recipientStatus(KycStatus.VERIFIED) + .provider("onfido") + .build(); + assertThat(result) + .usingRecursiveComparison() + .ignoringFields("kycResultId", "checkId", "providerRef", "checkedAt") + .isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(get(urlPathEqualTo("/v3.6/checks")) + .willReturn(aResponse().withStatus(400))); + + assertThatThrownBy(() -> kycProvider.verify(SENDER_ID, RECIPIENT_ID)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("KYC verification unavailable"); + + wireMock.verify(1, getRequestedFor(urlPathEqualTo("/v3.6/checks"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(get(urlPathEqualTo("/v3.6/checks")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> kycProvider.verify(SENDER_ID, RECIPIENT_ID)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("KYC verification unavailable"); + + wireMock.verify(3, getRequestedFor(urlPathEqualTo("/v3.6/checks"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + StringRedisTemplate stringRedisTemplate() { + return redisTemplate; + } + + @Bean + OnfidoProperties onfidoProperties() { + return new OnfidoProperties( + wireMock.baseUrl() + "/v3.6", + "test-token", + 2, + 60 + ); + } + + @Bean + OnfidoKycAdapter onfidoKycAdapter(OnfidoProperties properties, StringRedisTemplate template) { + return new OnfidoKycAdapter(properties, template); + } + } +} diff --git a/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapterRetryTest.java b/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapterRetryTest.java new file mode 100644 index 00000000..63f82696 --- /dev/null +++ b/compliance-travel-rule/compliance-travel-rule/src/test/java/com/stablecoin/payments/compliance/infrastructure/provider/worldcheck/WorldCheckSanctionsAdapterRetryTest.java @@ -0,0 +1,181 @@ +package com.stablecoin.payments.compliance.infrastructure.provider.worldcheck; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.compliance.domain.model.SanctionsResult; +import com.stablecoin.payments.compliance.domain.port.SanctionsProvider; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.util.List; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = WorldCheckSanctionsAdapterRetryTest.TestConfig.class) +@DisplayName("WorldCheckSanctionsAdapter retry behavior") +class WorldCheckSanctionsAdapterRetryTest { + + private static WireMockServer wireMock; + + @Autowired + private SanctionsProvider sanctionsProvider; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.sanctions.provider", () -> "world-check"); + registry.add("app.sanctions.world-check.base-url", () -> wireMock.baseUrl() + "/v2"); + registry.add("app.sanctions.world-check.api-key", () -> "test-key"); + registry.add("app.sanctions.world-check.api-secret", () -> "test-secret"); + registry.add("app.sanctions.world-check.group-id", () -> "test-group"); + registry.add("app.sanctions.world-check.timeout-seconds", () -> "2"); + registry.add("resilience4j.retry.instances.sanctions.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.sanctions.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.sanctions.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.sanctions.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/v2/cases/screeningRequest")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/v2/cases/screeningRequest")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "caseId": "test-id", + "caseSystemId": "WC-001", + "screeningState": "SCREENED", + "results": [] + } + """))); + + var senderId = UUID.fromString("11111111-1111-1111-1111-111111111111"); + var recipientId = UUID.fromString("22222222-2222-2222-2222-222222222222"); + var result = sanctionsProvider.screen(senderId, recipientId); + + var expected = SanctionsResult.builder() + .senderScreened(true) + .recipientScreened(true) + .senderHit(false) + .recipientHit(false) + .hitDetails(null) + .listsChecked(List.of("OFAC_SDN", "EU_CONSOLIDATED", "UN")) + .provider("world-check") + .build(); + assertThat(result) + .usingRecursiveComparison() + .ignoringFields("sanctionsResultId", "checkId", "providerRef", "screenedAt") + .isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/v2/cases/screeningRequest")) + .willReturn(aResponse().withStatus(400))); + + var senderId = UUID.fromString("11111111-1111-1111-1111-111111111111"); + var recipientId = UUID.fromString("22222222-2222-2222-2222-222222222222"); + + assertThatThrownBy(() -> sanctionsProvider.screen(senderId, recipientId)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Sanctions screening unavailable"); + + // Should call exactly once — 400 is not retried (sender call fails, no recipient call) + wireMock.verify(1, postRequestedFor(urlEqualTo("/v2/cases/screeningRequest"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/v2/cases/screeningRequest")) + .willReturn(aResponse().withStatus(503))); + + var senderId = UUID.fromString("11111111-1111-1111-1111-111111111111"); + var recipientId = UUID.fromString("22222222-2222-2222-2222-222222222222"); + + assertThatThrownBy(() -> sanctionsProvider.screen(senderId, recipientId)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Sanctions screening unavailable"); + + // max-attempts=3 => 3 calls for the first screenEntity call (sender) + wireMock.verify(3, postRequestedFor(urlEqualTo("/v2/cases/screeningRequest"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + WorldCheckProperties worldCheckProperties() { + return new WorldCheckProperties( + wireMock.baseUrl() + "/v2", + "test-key", + "test-secret", + "test-group", + 2 + ); + } + + @Bean + WorldCheckSanctionsAdapter worldCheckSanctionsAdapter(WorldCheckProperties properties) { + return new WorldCheckSanctionsAdapter(properties); + } + } +} diff --git a/fiat-off-ramp/fiat-off-ramp/build.gradle.kts b/fiat-off-ramp/fiat-off-ramp/build.gradle.kts index ac636f4b..158cd8f0 100644 --- a/fiat-off-ramp/fiat-off-ramp/build.gradle.kts +++ b/fiat-off-ramp/fiat-off-ramp/build.gradle.kts @@ -97,6 +97,7 @@ dependencies { // Resilience4j implementation("io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion") implementation("io.github.resilience4j:resilience4j-circuitbreaker:$resilience4jVersion") + implementation("io.github.resilience4j:resilience4j-retry:$resilience4jVersion") // MapStruct (compiler args set below in JavaCompile task) implementation("org.mapstruct:mapstruct:$mapstructVersion") diff --git a/fiat-off-ramp/fiat-off-ramp/src/integration-test/resources/application-integration-test.yml b/fiat-off-ramp/fiat-off-ramp/src/integration-test/resources/application-integration-test.yml index eee44a98..d204d50d 100644 --- a/fiat-off-ramp/fiat-off-ramp/src/integration-test/resources/application-integration-test.yml +++ b/fiat-off-ramp/fiat-off-ramp/src/integration-test/resources/application-integration-test.yml @@ -35,6 +35,14 @@ app: monitor: enabled: false +resilience4j: + retry: + instances: + circle: + wait-duration: 10ms + modulr: + wait-duration: 10ms + logging: level: com.stablecoin.payments: DEBUG diff --git a/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapter.java b/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapter.java index 96e743b5..2466e33c 100644 --- a/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapter.java +++ b/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapter.java @@ -3,8 +3,8 @@ import com.stablecoin.payments.offramp.domain.port.RedemptionGateway; import com.stablecoin.payments.offramp.domain.port.RedemptionRequest; import com.stablecoin.payments.offramp.domain.port.RedemptionResult; -import io.github.resilience4j.circuitbreaker.CallNotPermittedException; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -50,6 +50,7 @@ public CircleRedemptionAdapter(CircleProperties properties) { } @Override + @Retry(name = "circle", fallbackMethod = "redeemFallback") @CircuitBreaker(name = "circle", fallbackMethod = "redeemFallback") public RedemptionResult redeem(RedemptionRequest request) { log.info("[CIRCLE] Redeeming stablecoin payoutId={} stablecoin={} amount={}", @@ -87,9 +88,9 @@ public RedemptionResult redeem(RedemptionRequest request) { } @SuppressWarnings("unused") - private RedemptionResult redeemFallback(RedemptionRequest request, CallNotPermittedException ex) { - log.error("[CIRCLE] Circuit breaker open — redemption failed payoutId={}", - request.payoutId(), ex); + private RedemptionResult redeemFallback(RedemptionRequest request, Exception ex) { + log.error("[CIRCLE] Resilience fallback — redemption failed payoutId={} due to {}", + request.payoutId(), ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Circle redemption unavailable", ex); } } diff --git a/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapter.java b/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapter.java index ca31da2a..33a82c11 100644 --- a/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapter.java +++ b/fiat-off-ramp/fiat-off-ramp/src/main/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapter.java @@ -5,8 +5,8 @@ import com.stablecoin.payments.offramp.domain.port.PayoutPartnerGateway; import com.stablecoin.payments.offramp.domain.port.PayoutRequest; import com.stablecoin.payments.offramp.domain.port.PayoutResult; -import io.github.resilience4j.circuitbreaker.CallNotPermittedException; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -57,6 +57,7 @@ public ModulrPayoutAdapter(ModulrProperties properties) { } @Override + @Retry(name = "modulr", fallbackMethod = "initiatePayoutFallback") @CircuitBreaker(name = "modulr", fallbackMethod = "initiatePayoutFallback") public PayoutResult initiatePayout(PayoutRequest request) { log.info("[MODULR] Initiating SEPA payout payoutId={} amount={} currency={}", @@ -108,9 +109,9 @@ private static String resolvePermittedScheme(PaymentRail rail) { } @SuppressWarnings("unused") - private PayoutResult initiatePayoutFallback(PayoutRequest request, CallNotPermittedException ex) { - log.error("[MODULR] Circuit breaker open — payout failed payoutId={}", - request.payoutId(), ex); + private PayoutResult initiatePayoutFallback(PayoutRequest request, Exception ex) { + log.error("[MODULR] Resilience fallback — payout failed payoutId={} due to {}", + request.payoutId(), ex.getClass().getSimpleName(), ex); throw new PayoutPartnerException( request.payoutId(), "modulr", "Modulr payout unavailable", ex); } diff --git a/fiat-off-ramp/fiat-off-ramp/src/main/resources/application.yml b/fiat-off-ramp/fiat-off-ramp/src/main/resources/application.yml index 46d11286..31a53554 100644 --- a/fiat-off-ramp/fiat-off-ramp/src/main/resources/application.yml +++ b/fiat-off-ramp/fiat-off-ramp/src/main/resources/application.yml @@ -135,6 +135,41 @@ app: webhook-secret: ${MODULR_WEBHOOK_SECRET:} tolerance-seconds: 300 +resilience4j: + circuitbreaker: + circuit-breaker-aspect-order: 1 + retry: + retry-aspect-order: 3 + instances: + circle: + max-attempts: 3 + wait-duration: 1s + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + modulr: + max-attempts: 3 + wait-duration: 1s + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + logging: level: com.stablecoin.payments: INFO diff --git a/fiat-off-ramp/fiat-off-ramp/src/test/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapterRetryTest.java b/fiat-off-ramp/fiat-off-ramp/src/test/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapterRetryTest.java new file mode 100644 index 00000000..da6ad489 --- /dev/null +++ b/fiat-off-ramp/fiat-off-ramp/src/test/java/com/stablecoin/payments/offramp/infrastructure/provider/circle/CircleRedemptionAdapterRetryTest.java @@ -0,0 +1,175 @@ +package com.stablecoin.payments.offramp.infrastructure.provider.circle; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.offramp.domain.port.RedemptionGateway; +import com.stablecoin.payments.offramp.domain.port.RedemptionRequest; +import com.stablecoin.payments.offramp.domain.port.RedemptionResult; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.math.BigDecimal; +import java.time.Instant; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = CircleRedemptionAdapterRetryTest.TestConfig.class) +@DisplayName("CircleRedemptionAdapter retry behavior") +class CircleRedemptionAdapterRetryTest { + + private static WireMockServer wireMock; + private static final UUID PAYOUT_ID = UUID.fromString("a1b2c3d4-e5f6-7890-abcd-ef1234567890"); + + @Autowired + private RedemptionGateway redemptionGateway; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.redemption.provider", () -> "circle"); + registry.add("resilience4j.retry.instances.circle.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.circle.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.circle.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.circle.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + private RedemptionRequest aRedemptionRequest() { + return new RedemptionRequest(PAYOUT_ID, "USDC", new BigDecimal("10000.000000"), BigDecimal.ONE); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/v1/businessAccount/payouts")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/v1/businessAccount/payouts")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(201) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "data": { + "id": "circle-payout-ref-001", + "amount": { + "amount": "10000.00", + "currency": "USD" + }, + "status": "pending", + "createDate": "2026-03-10T12:00:00.000Z" + } + } + """))); + + var result = redemptionGateway.redeem(aRedemptionRequest()); + + var expected = new RedemptionResult( + "circle-payout-ref-001", + new BigDecimal("10000.00"), + "USD", + Instant.parse("2026-03-10T12:00:00.000Z") + ); + assertThat(result).usingRecursiveComparison().isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/v1/businessAccount/payouts")) + .willReturn(aResponse() + .withStatus(400) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "code": 2, + "message": "Invalid idempotency key" + } + """))); + + assertThatThrownBy(() -> redemptionGateway.redeem(aRedemptionRequest())) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Circle redemption unavailable"); + + wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/businessAccount/payouts"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/v1/businessAccount/payouts")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> redemptionGateway.redeem(aRedemptionRequest())) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Circle redemption unavailable"); + + wireMock.verify(3, postRequestedFor(urlEqualTo("/v1/businessAccount/payouts"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + CircleProperties circleProperties() { + return new CircleProperties( + wireMock.baseUrl(), "SAND_API_KEY_TEST", "wire-bank-001", 10); + } + + @Bean + CircleRedemptionAdapter circleRedemptionAdapter(CircleProperties properties) { + return new CircleRedemptionAdapter(properties); + } + } +} diff --git a/fiat-off-ramp/fiat-off-ramp/src/test/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapterRetryTest.java b/fiat-off-ramp/fiat-off-ramp/src/test/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapterRetryTest.java new file mode 100644 index 00000000..f8306870 --- /dev/null +++ b/fiat-off-ramp/fiat-off-ramp/src/test/java/com/stablecoin/payments/offramp/infrastructure/provider/modulr/ModulrPayoutAdapterRetryTest.java @@ -0,0 +1,178 @@ +package com.stablecoin.payments.offramp.infrastructure.provider.modulr; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.offramp.domain.exception.PayoutPartnerException; +import com.stablecoin.payments.offramp.domain.model.AccountType; +import com.stablecoin.payments.offramp.domain.model.BankAccount; +import com.stablecoin.payments.offramp.domain.model.PartnerIdentifier; +import com.stablecoin.payments.offramp.domain.model.PaymentRail; +import com.stablecoin.payments.offramp.domain.port.PayoutPartnerGateway; +import com.stablecoin.payments.offramp.domain.port.PayoutRequest; +import com.stablecoin.payments.offramp.domain.port.PayoutResult; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.math.BigDecimal; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = ModulrPayoutAdapterRetryTest.TestConfig.class) +@DisplayName("ModulrPayoutAdapter retry behavior") +class ModulrPayoutAdapterRetryTest { + + private static WireMockServer wireMock; + private static final UUID PAYOUT_ID = UUID.fromString("887adb57-1d2e-4f3a-b5c6-d7e8f9a0b1c2"); + + @Autowired + private PayoutPartnerGateway payoutPartnerGateway; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.payout.provider", () -> "modulr"); + registry.add("resilience4j.retry.instances.modulr.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.modulr.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.modulr.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.modulr.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + private PayoutRequest aPayoutRequest() { + return new PayoutRequest( + PAYOUT_ID, + new BigDecimal("9500.00"), + "EUR", + new BankAccount("DE89370400440532013000", "DEUTDEFF", AccountType.IBAN, "DE"), + null, + PaymentRail.SEPA, + new PartnerIdentifier("modulr-001", "modulr") + ); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/api-sandbox-token/payments")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/api-sandbox-token/payments")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "id": "P120003AQM", + "status": "VALIDATED", + "createdDate": "2026-03-15T10:30:00.000+0000", + "externalReference": "887adb57-1d2e-4f3a-b5c6-d7e8f9a0b1c2", + "approvalStatus": "NOTNEEDED", + "message": "" + } + """))); + + var result = payoutPartnerGateway.initiatePayout(aPayoutRequest()); + + var expected = new PayoutResult("P120003AQM", "VALIDATED", null); + assertThat(result).usingRecursiveComparison().isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/api-sandbox-token/payments")) + .willReturn(aResponse() + .withStatus(400) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "field": "amount", + "code": "INVALID", + "message": "Amount must be positive" + } + """))); + + assertThatThrownBy(() -> payoutPartnerGateway.initiatePayout(aPayoutRequest())) + .isInstanceOf(PayoutPartnerException.class); + + wireMock.verify(1, postRequestedFor(urlEqualTo("/api-sandbox-token/payments"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/api-sandbox-token/payments")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> payoutPartnerGateway.initiatePayout(aPayoutRequest())) + .isInstanceOf(PayoutPartnerException.class); + + wireMock.verify(3, postRequestedFor(urlEqualTo("/api-sandbox-token/payments"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + ModulrProperties modulrProperties() { + return new ModulrProperties( + wireMock.baseUrl(), "SANDBOX_TEST_API_KEY", "test-secret", "A1100ABCD1", 10); + } + + @Bean + ModulrPayoutAdapter modulrPayoutAdapter(ModulrProperties properties) { + return new ModulrPayoutAdapter(properties); + } + } +} diff --git a/fiat-on-ramp/fiat-on-ramp/build.gradle.kts b/fiat-on-ramp/fiat-on-ramp/build.gradle.kts index 79ff23e8..511b0d32 100644 --- a/fiat-on-ramp/fiat-on-ramp/build.gradle.kts +++ b/fiat-on-ramp/fiat-on-ramp/build.gradle.kts @@ -97,6 +97,7 @@ dependencies { // Resilience4j implementation("io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion") implementation("io.github.resilience4j:resilience4j-circuitbreaker:$resilience4jVersion") + implementation("io.github.resilience4j:resilience4j-retry:$resilience4jVersion") // MapStruct (compiler args set below in JavaCompile task) implementation("org.mapstruct:mapstruct:$mapstructVersion") diff --git a/fiat-on-ramp/fiat-on-ramp/src/integration-test/resources/application-integration-test.yml b/fiat-on-ramp/fiat-on-ramp/src/integration-test/resources/application-integration-test.yml index 7e70206a..dc2ab049 100644 --- a/fiat-on-ramp/fiat-on-ramp/src/integration-test/resources/application-integration-test.yml +++ b/fiat-on-ramp/fiat-on-ramp/src/integration-test/resources/application-integration-test.yml @@ -34,6 +34,12 @@ app: psp: provider: dev +resilience4j: + retry: + instances: + stripe: + wait-duration: 10ms + logging: level: com.stablecoin.payments: DEBUG diff --git a/fiat-on-ramp/fiat-on-ramp/src/main/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapter.java b/fiat-on-ramp/fiat-on-ramp/src/main/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapter.java index 7357e6a9..7ee7baaa 100644 --- a/fiat-on-ramp/fiat-on-ramp/src/main/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapter.java +++ b/fiat-on-ramp/fiat-on-ramp/src/main/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapter.java @@ -6,6 +6,7 @@ import com.stablecoin.payments.onramp.domain.port.PspRefundRequest; import com.stablecoin.payments.onramp.domain.port.PspRefundResult; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -46,6 +47,7 @@ public StripePspAdapter(StripeProperties properties) { } @Override + @Retry(name = "stripe", fallbackMethod = "initiatePaymentFallback") @CircuitBreaker(name = "stripe", fallbackMethod = "initiatePaymentFallback") public PspPaymentResult initiatePayment(PspPaymentRequest request) { log.info("[STRIPE] Initiating payment collectionId={} amount={} currency={}", @@ -76,6 +78,7 @@ public PspPaymentResult initiatePayment(PspPaymentRequest request) { } @Override + @Retry(name = "stripe", fallbackMethod = "initiateRefundFallback") @CircuitBreaker(name = "stripe", fallbackMethod = "initiateRefundFallback") public PspRefundResult initiateRefund(PspRefundRequest request) { log.info("[STRIPE] Initiating refund collectionId={} pspRef={} amount={}", @@ -102,15 +105,15 @@ public PspRefundResult initiateRefund(PspRefundRequest request) { @SuppressWarnings("unused") private PspPaymentResult initiatePaymentFallback(PspPaymentRequest request, Exception ex) { - log.error("[STRIPE] Circuit breaker open — payment initiation failed collectionId={}", - request.collectionId(), ex); + log.error("[STRIPE] Resilience fallback — payment initiation failed collectionId={} due to {}", + request.collectionId(), ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Stripe payment initiation unavailable", ex); } @SuppressWarnings("unused") private PspRefundResult initiateRefundFallback(PspRefundRequest request, Exception ex) { - log.error("[STRIPE] Circuit breaker open — refund initiation failed collectionId={}", - request.collectionId(), ex); + log.error("[STRIPE] Resilience fallback — refund initiation failed collectionId={} due to {}", + request.collectionId(), ex.getClass().getSimpleName(), ex); throw new IllegalStateException("Stripe refund initiation unavailable", ex); } diff --git a/fiat-on-ramp/fiat-on-ramp/src/main/resources/application.yml b/fiat-on-ramp/fiat-on-ramp/src/main/resources/application.yml index 3426fcb0..4f0317be 100644 --- a/fiat-on-ramp/fiat-on-ramp/src/main/resources/application.yml +++ b/fiat-on-ramp/fiat-on-ramp/src/main/resources/application.yml @@ -125,6 +125,27 @@ app: webhook-secret: ${STRIPE_WEBHOOK_SECRET:whsec_test_default} tolerance-seconds: 300 +resilience4j: + circuitbreaker: + circuit-breaker-aspect-order: 1 + retry: + retry-aspect-order: 3 + instances: + stripe: + max-attempts: 3 + wait-duration: 1s + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + logging: level: com.stablecoin.payments: INFO diff --git a/fiat-on-ramp/fiat-on-ramp/src/test/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapterRetryTest.java b/fiat-on-ramp/fiat-on-ramp/src/test/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapterRetryTest.java new file mode 100644 index 00000000..650b6bb5 --- /dev/null +++ b/fiat-on-ramp/fiat-on-ramp/src/test/java/com/stablecoin/payments/onramp/infrastructure/provider/stripe/StripePspAdapterRetryTest.java @@ -0,0 +1,184 @@ +package com.stablecoin.payments.onramp.infrastructure.provider.stripe; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.onramp.domain.model.AccountType; +import com.stablecoin.payments.onramp.domain.model.BankAccount; +import com.stablecoin.payments.onramp.domain.model.Money; +import com.stablecoin.payments.onramp.domain.model.PaymentRail; +import com.stablecoin.payments.onramp.domain.model.PaymentRailType; +import com.stablecoin.payments.onramp.domain.port.PspGateway; +import com.stablecoin.payments.onramp.domain.port.PspPaymentRequest; +import com.stablecoin.payments.onramp.domain.port.PspPaymentResult; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.math.BigDecimal; +import java.util.UUID; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(classes = StripePspAdapterRetryTest.TestConfig.class) +@DisplayName("StripePspAdapter retry behavior") +class StripePspAdapterRetryTest { + + private static WireMockServer wireMock; + private static final UUID COLLECTION_ID = UUID.fromString("a1b2c3d4-e5f6-7890-abcd-ef1234567890"); + + @Autowired + private PspGateway pspGateway; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.psp.provider", () -> "stripe"); + registry.add("resilience4j.retry.instances.stripe.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.stripe.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.stripe.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.stripe.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + private PspPaymentRequest aPaymentRequest() { + return new PspPaymentRequest( + COLLECTION_ID, + new Money(new BigDecimal("250.00"), "USD"), + new PaymentRail(PaymentRailType.ACH, "US", "USD"), + new BankAccount("hash123", "021000021", AccountType.ACH_ROUTING, "US"), + "stripe", + COLLECTION_ID.toString() + ); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + wireMock.stubFor(post(urlEqualTo("/v1/payment_intents")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(post(urlEqualTo("/v1/payment_intents")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("third-attempt")); + + wireMock.stubFor(post(urlEqualTo("/v1/payment_intents")) + .inScenario("retry-then-success") + .whenScenarioStateIs("third-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "id": "pi_test123", + "status": "succeeded", + "amount": 25000, + "currency": "usd", + "client_secret": "pi_test123_secret_xxx" + } + """))); + + var result = pspGateway.initiatePayment(aPaymentRequest()); + + var expected = new PspPaymentResult("pi_test123", "succeeded"); + assertThat(result).usingRecursiveComparison().isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 402 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(post(urlEqualTo("/v1/payment_intents")) + .willReturn(aResponse() + .withStatus(402) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "error": { + "type": "card_error", + "message": "Your card was declined." + } + } + """))); + + assertThatThrownBy(() -> pspGateway.initiatePayment(aPaymentRequest())) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Stripe payment initiation unavailable"); + + wireMock.verify(1, postRequestedFor(urlEqualTo("/v1/payment_intents"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(post(urlEqualTo("/v1/payment_intents")) + .willReturn(aResponse().withStatus(503))); + + assertThatThrownBy(() -> pspGateway.initiatePayment(aPaymentRequest())) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Stripe payment initiation unavailable"); + + wireMock.verify(3, postRequestedFor(urlEqualTo("/v1/payment_intents"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + StripeProperties stripeProperties() { + return new StripeProperties(wireMock.baseUrl(), "sk_test_xxx", 10); + } + + @Bean + StripePspAdapter stripePspAdapter(StripeProperties properties) { + return new StripePspAdapter(properties); + } + } +} diff --git a/fx-liquidity-engine/fx-liquidity-engine/build.gradle.kts b/fx-liquidity-engine/fx-liquidity-engine/build.gradle.kts index 82465c09..86ae72a7 100644 --- a/fx-liquidity-engine/fx-liquidity-engine/build.gradle.kts +++ b/fx-liquidity-engine/fx-liquidity-engine/build.gradle.kts @@ -101,6 +101,7 @@ dependencies { // Resilience4j implementation("io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion") implementation("io.github.resilience4j:resilience4j-circuitbreaker:$resilience4jVersion") + implementation("io.github.resilience4j:resilience4j-retry:$resilience4jVersion") // MapStruct implementation("org.mapstruct:mapstruct:$mapstructVersion") diff --git a/fx-liquidity-engine/fx-liquidity-engine/src/integration-test/resources/application-integration-test.yml b/fx-liquidity-engine/fx-liquidity-engine/src/integration-test/resources/application-integration-test.yml index 388cb564..bc6e1cc3 100644 --- a/fx-liquidity-engine/fx-liquidity-engine/src/integration-test/resources/application-integration-test.yml +++ b/fx-liquidity-engine/fx-liquidity-engine/src/integration-test/resources/application-integration-test.yml @@ -37,6 +37,12 @@ app: lock-expiry: enabled: false +resilience4j: + retry: + instances: + fxRate: + wait-duration: 10ms + logging: level: com.stablecoin.payments: DEBUG diff --git a/fx-liquidity-engine/fx-liquidity-engine/src/main/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapter.java b/fx-liquidity-engine/fx-liquidity-engine/src/main/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapter.java index 30bf5913..91d1602b 100644 --- a/fx-liquidity-engine/fx-liquidity-engine/src/main/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapter.java +++ b/fx-liquidity-engine/fx-liquidity-engine/src/main/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapter.java @@ -3,6 +3,7 @@ import com.stablecoin.payments.fx.domain.model.CorridorRate; import com.stablecoin.payments.fx.domain.port.RateProvider; import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -42,6 +43,7 @@ public RefinitivRateAdapter(RefinitivProperties properties) { } @Override + @Retry(name = "fxRate", fallbackMethod = "getRateFallback") @CircuitBreaker(name = "fxRate", fallbackMethod = "getRateFallback") public Optional getRate(String fromCurrency, String toCurrency) { log.info("[REFINITIV] Fetching rate for {}:{}", fromCurrency, toCurrency); @@ -84,7 +86,8 @@ public String providerName() { @SuppressWarnings("unused") private Optional getRateFallback(String fromCurrency, String toCurrency, Exception ex) { - log.error("[REFINITIV] Circuit breaker open for {}:{}", fromCurrency, toCurrency, ex); + log.error("[REFINITIV] Resilience fallback for {}:{} due to {}", + fromCurrency, toCurrency, ex.getClass().getSimpleName(), ex); return Optional.empty(); } } diff --git a/fx-liquidity-engine/fx-liquidity-engine/src/main/resources/application.yml b/fx-liquidity-engine/fx-liquidity-engine/src/main/resources/application.yml index 48fb5e99..8415898c 100644 --- a/fx-liquidity-engine/fx-liquidity-engine/src/main/resources/application.yml +++ b/fx-liquidity-engine/fx-liquidity-engine/src/main/resources/application.yml @@ -124,6 +124,27 @@ app: enabled: ${LOCK_EXPIRY_ENABLED:true} interval-ms: ${LOCK_EXPIRY_INTERVAL_MS:5000} +resilience4j: + circuitbreaker: + circuit-breaker-aspect-order: 1 + retry: + retry-aspect-order: 3 + instances: + fxRate: + max-attempts: 3 + wait-duration: 200ms + enable-exponential-backoff: true + exponential-backoff-multiplier: 2 + retry-exceptions: + - java.io.IOException + - java.net.http.HttpTimeoutException + - java.net.http.HttpConnectTimeoutException + - org.springframework.web.client.ResourceAccessException + - org.springframework.web.client.HttpServerErrorException + ignore-exceptions: + - org.springframework.web.client.HttpClientErrorException + - io.github.resilience4j.circuitbreaker.CallNotPermittedException + logging: level: com.stablecoin.payments: INFO diff --git a/fx-liquidity-engine/fx-liquidity-engine/src/test/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapterRetryTest.java b/fx-liquidity-engine/fx-liquidity-engine/src/test/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapterRetryTest.java new file mode 100644 index 00000000..a17a11e3 --- /dev/null +++ b/fx-liquidity-engine/fx-liquidity-engine/src/test/java/com/stablecoin/payments/fx/infrastructure/provider/refinitiv/RefinitivRateAdapterRetryTest.java @@ -0,0 +1,169 @@ +package com.stablecoin.payments.fx.infrastructure.provider.refinitiv; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.stablecoin.payments.fx.domain.model.CorridorRate; +import com.stablecoin.payments.fx.domain.port.RateProvider; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration; +import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.EnableAspectJAutoProxy; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import java.math.BigDecimal; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(classes = RefinitivRateAdapterRetryTest.TestConfig.class) +@DisplayName("RefinitivRateAdapter retry behavior") +class RefinitivRateAdapterRetryTest { + + /** Must match RefinitivRateAdapter.DEFAULT_SPREAD_BPS */ + private static final int EXPECTED_SPREAD_BPS = 30; + /** Must match RefinitivRateAdapter.DEFAULT_FEE_BPS */ + private static final int EXPECTED_FEE_BPS = 30; + + private static WireMockServer wireMock; + + @Autowired + private RateProvider rateProvider; + + @Autowired + private CircuitBreakerRegistry circuitBreakerRegistry; + + @BeforeAll + static void startWireMock() { + wireMock = new WireMockServer(wireMockConfig().dynamicPort()); + wireMock.start(); + } + + @AfterAll + static void stopWireMock() { + wireMock.stop(); + } + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("app.fx.rate-provider", () -> "refinitiv"); + registry.add("resilience4j.retry.instances.fxRate.max-attempts", () -> "3"); + registry.add("resilience4j.retry.instances.fxRate.wait-duration", () -> "10ms"); + registry.add("resilience4j.retry.instances.fxRate.retry-exceptions[0]", + () -> "org.springframework.web.client.HttpServerErrorException"); + registry.add("resilience4j.retry.instances.fxRate.ignore-exceptions[0]", + () -> "org.springframework.web.client.HttpClientErrorException"); + registry.add("resilience4j.retry.retry-aspect-order", () -> "3"); + registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1"); + } + + @BeforeEach + void setUp() { + wireMock.resetAll(); + circuitBreakerRegistry.getAllCircuitBreakers() + .forEach(cb -> cb.transitionToClosedState()); + } + + @Test + @DisplayName("should retry on transient 503 failure then succeed") + void shouldRetryOnTransientFailureThenSucceed() { + long nowMs = System.currentTimeMillis(); + + wireMock.stubFor(get(urlEqualTo("/data/pricing/v1/rates/USDEUR")) + .inScenario("retry-then-success") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn(aResponse().withStatus(503)) + .willSetStateTo("second-attempt")); + + wireMock.stubFor(get(urlEqualTo("/data/pricing/v1/rates/USDEUR")) + .inScenario("retry-then-success") + .whenScenarioStateIs("second-attempt") + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(""" + { + "currencyPair": "USDEUR", + "mid": 0.9200000000, + "bid": 0.9195000000, + "ask": 0.9205000000, + "timestamp": %d + } + """.formatted(nowMs)))); + + var result = rateProvider.getRate("USD", "EUR"); + + assertThat(result).isPresent(); + var expected = CorridorRate.builder() + .fromCurrency("USD") + .toCurrency("EUR") + .rate(new BigDecimal("0.92")) + .spreadBps(EXPECTED_SPREAD_BPS) + .feeBps(EXPECTED_FEE_BPS) + .provider("refinitiv") + .build(); + assertThat(result.get()) + .usingRecursiveComparison() + .withComparatorForType(BigDecimal::compareTo, BigDecimal.class) + .ignoringFields("ageMs") + .isEqualTo(expected); + } + + @Test + @DisplayName("should not retry on 400 client error") + void shouldNotRetryOnClientError() { + wireMock.stubFor(get(urlEqualTo("/data/pricing/v1/rates/USDEUR")) + .willReturn(aResponse().withStatus(400))); + + // 400 throws HttpClientErrorException which is in ignore-exceptions — no retry + // The fallback returns Optional.empty() instead of throwing + var result = rateProvider.getRate("USD", "EUR"); + assertThat(result).isEmpty(); + + wireMock.verify(1, getRequestedFor(urlEqualTo("/data/pricing/v1/rates/USDEUR"))); + } + + @Test + @DisplayName("should exhaust retries on persistent 503 and invoke fallback") + void shouldExhaustRetriesAndInvokeFallback() { + wireMock.stubFor(get(urlEqualTo("/data/pricing/v1/rates/USDEUR")) + .willReturn(aResponse().withStatus(503))); + + // Fallback returns Optional.empty() + var result = rateProvider.getRate("USD", "EUR"); + assertThat(result).isEmpty(); + + wireMock.verify(3, getRequestedFor(urlEqualTo("/data/pricing/v1/rates/USDEUR"))); + } + + @Configuration + @EnableAspectJAutoProxy + @Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class}) + static class TestConfig { + + @Bean + RefinitivProperties refinitivProperties() { + return new RefinitivProperties( + wireMock.baseUrl(), "test-api-key", 2); + } + + @Bean + RefinitivRateAdapter refinitivRateAdapter(RefinitivProperties properties) { + return new RefinitivRateAdapter(properties); + } + } +}