Skip to content

Commit 1b797b3

Browse files
Puneethkumarckclaude
Puneethkumarck
and
claude
authored
feat(infra): add Resilience4j retry to all external adapters (STA-117) (#186)
* feat(infra): add Resilience4j retry to all external adapters (STA-117) Add @Retry annotation alongside existing @CIRCUITBREAKER on all 10 external service adapters across 5 services. Decorator ordering ensures Retry (outer, order=3) wraps CircuitBreaker (inner, order=1). Changes per service: - S2 Compliance: WorldCheck, Onfido, Chainalysis adapters - S6 FX: Refinitiv rate adapter (exponential backoff) - S3 On-Ramp: Stripe PSP adapter (payment + refund methods) - S4 Blockchain: Fireblocks, EVM RPC, Solana RPC adapters - S5 Off-Ramp: Circle, Modulr adapters (fallback params widened from CallNotPermittedException to Exception) Each service includes: - YAML retry config with retry-exceptions/ignore-exceptions - resilience4j-retry Gradle dependency - 3 retry behavior tests per adapter (30 total) - IT YAML overrides with wait-duration=10ms Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(infra): address CodeRabbit review — neutral fallback logs, tighter assertions (STA-117) - Replace Circuit breaker open with Resilience fallback in all 10 adapter fallback log messages since fallbacks now serve both Retry and CircuitBreaker - Tighten all retry test exception assertions from Exception.class to concrete types (IllegalStateException/PayoutPartnerException) with exact message checks - Add 3 getTransactionStatus retry tests for Fireblocks (GET endpoint coverage) - Extract magic spreadBps/feeBps numbers to named constants in Refinitiv test Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix(infra): fix import ordering in ModulrPayoutAdapterRetryTest (STA-117) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 9eac975 commit 1b797b3

35 files changed

Lines changed: 2056 additions & 34 deletions

File tree

‎blockchain-custody/blockchain-custody/build.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ dependencies {
100100
// Resilience4j
101101
implementation("io.github.resilience4j:resilience4j-spring-boot3:$resilience4jVersion")
102102
implementation("io.github.resilience4j:resilience4j-circuitbreaker:$resilience4jVersion")
103+
implementation("io.github.resilience4j:resilience4j-retry:$resilience4jVersion")
103104

104105
// Dev custody adapter — EVM transaction signing
105106
implementation("org.web3j:core:$web3jVersion") {

‎blockchain-custody/blockchain-custody/src/integration-test/resources/application-integration-test.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@ app:
3434
fallback-adapters:
3535
enabled: true
3636

37+
resilience4j:
38+
retry:
39+
instances:
40+
fireblocks:
41+
wait-duration: 10ms
42+
evmRpc:
43+
wait-duration: 10ms
44+
solanaRpc:
45+
wait-duration: 10ms
46+
3747
logging:
3848
level:
3949
com.stablecoin.payments: DEBUG

‎blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapter.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.stablecoin.payments.custody.domain.port.ChainRpcProvider;
55
import com.stablecoin.payments.custody.domain.port.TransactionReceipt;
66
import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
7+
import io.github.resilience4j.retry.annotation.Retry;
78
import lombok.extern.slf4j.Slf4j;
89
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
910
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -61,6 +62,7 @@ public EvmRpcAdapter(EvmChainProperties properties) {
6162
}
6263

6364
@Override
65+
@Retry(name = "evmRpc", fallbackMethod = "getTransactionReceiptFallback")
6466
@CircuitBreaker(name = "evmRpc", fallbackMethod = "getTransactionReceiptFallback")
6567
public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash) {
6668
log.info("[EVM-RPC] Getting transaction receipt chain={} txHash={}", chainId.value(), txHash);
@@ -88,6 +90,7 @@ public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash)
8890
}
8991

9092
@Override
93+
@Retry(name = "evmRpc", fallbackMethod = "getLatestBlockNumberFallback")
9194
@CircuitBreaker(name = "evmRpc", fallbackMethod = "getLatestBlockNumberFallback")
9295
public long getLatestBlockNumber(ChainId chainId) {
9396
log.info("[EVM-RPC] Getting latest block number chain={}", chainId.value());
@@ -100,6 +103,7 @@ public long getLatestBlockNumber(ChainId chainId) {
100103
}
101104

102105
@Override
106+
@Retry(name = "evmRpc", fallbackMethod = "getTokenBalanceFallback")
103107
@CircuitBreaker(name = "evmRpc", fallbackMethod = "getTokenBalanceFallback")
104108
public BigDecimal getTokenBalance(ChainId chainId, String address, String tokenContract) {
105109
log.info("[EVM-RPC] Getting token balance chain={} address={} contract={}",
@@ -177,23 +181,23 @@ static String encodeBalanceOfCall(String address) {
177181

178182
@SuppressWarnings("unused")
179183
private TransactionReceipt getTransactionReceiptFallback(ChainId chainId, String txHash, Exception ex) {
180-
log.error("[EVM-RPC] Circuit breaker open — getTransactionReceipt failed chain={} txHash={}",
181-
chainId.value(), txHash, ex);
184+
log.error("[EVM-RPC] Resilience fallback — getTransactionReceipt failed chain={} txHash={} due to {}",
185+
chainId.value(), txHash, ex.getClass().getSimpleName(), ex);
182186
throw new IllegalStateException("EVM RPC unavailable for getTransactionReceipt", ex);
183187
}
184188

185189
@SuppressWarnings("unused")
186190
private long getLatestBlockNumberFallback(ChainId chainId, Exception ex) {
187-
log.error("[EVM-RPC] Circuit breaker open — getLatestBlockNumber failed chain={}",
188-
chainId.value(), ex);
191+
log.error("[EVM-RPC] Resilience fallback — getLatestBlockNumber failed chain={} due to {}",
192+
chainId.value(), ex.getClass().getSimpleName(), ex);
189193
throw new IllegalStateException("EVM RPC unavailable for getLatestBlockNumber", ex);
190194
}
191195

192196
@SuppressWarnings("unused")
193197
private BigDecimal getTokenBalanceFallback(ChainId chainId, String address, String tokenContract,
194198
Exception ex) {
195-
log.error("[EVM-RPC] Circuit breaker open — getTokenBalance failed chain={} address={}",
196-
chainId.value(), address, ex);
199+
log.error("[EVM-RPC] Resilience fallback — getTokenBalance failed chain={} address={} due to {}",
200+
chainId.value(), address, ex.getClass().getSimpleName(), ex);
197201
throw new IllegalStateException("EVM RPC unavailable for getTokenBalance", ex);
198202
}
199203

‎blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/fireblocks/FireblocksCustodyAdapter.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import com.stablecoin.payments.custody.domain.port.SignResult;
88
import com.stablecoin.payments.custody.domain.port.TransactionStatus;
99
import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
10+
import io.github.resilience4j.retry.annotation.Retry;
1011
import lombok.extern.slf4j.Slf4j;
1112
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
1213
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -68,6 +69,7 @@ public FireblocksCustodyAdapter(FireblocksProperties properties) {
6869
}
6970

7071
@Override
72+
@Retry(name = "fireblocks", fallbackMethod = "signAndSubmitFallback")
7173
@CircuitBreaker(name = "fireblocks", fallbackMethod = "signAndSubmitFallback")
7274
public SignResult signAndSubmit(SignRequest request) {
7375
log.info("[FIREBLOCKS] Signing and submitting transfer transferId={} chain={} to={}",
@@ -109,6 +111,7 @@ public SignResult signAndSubmit(SignRequest request) {
109111
}
110112

111113
@Override
114+
@Retry(name = "fireblocks", fallbackMethod = "getTransactionStatusFallback")
112115
@CircuitBreaker(name = "fireblocks", fallbackMethod = "getTransactionStatusFallback")
113116
public TransactionStatus getTransactionStatus(String txId) {
114117
log.info("[FIREBLOCKS] Getting transaction status txId={}", txId);
@@ -131,14 +134,15 @@ public TransactionStatus getTransactionStatus(String txId) {
131134

132135
@SuppressWarnings("unused")
133136
private SignResult signAndSubmitFallback(SignRequest request, Exception ex) {
134-
log.error("[FIREBLOCKS] Circuit breaker open — signAndSubmit failed transferId={}",
135-
request.transferId(), ex);
137+
log.error("[FIREBLOCKS] Resilience fallback — signAndSubmit failed transferId={} due to {}",
138+
request.transferId(), ex.getClass().getSimpleName(), ex);
136139
throw new IllegalStateException("Fireblocks custody unavailable", ex);
137140
}
138141

139142
@SuppressWarnings("unused")
140143
private TransactionStatus getTransactionStatusFallback(String txId, Exception ex) {
141-
log.error("[FIREBLOCKS] Circuit breaker open — getTransactionStatus failed txId={}", txId, ex);
144+
log.error("[FIREBLOCKS] Resilience fallback — getTransactionStatus failed txId={} due to {}",
145+
txId, ex.getClass().getSimpleName(), ex);
142146
throw new IllegalStateException("Fireblocks custody unavailable", ex);
143147
}
144148

‎blockchain-custody/blockchain-custody/src/main/java/com/stablecoin/payments/custody/infrastructure/provider/solana/SolanaRpcAdapter.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.stablecoin.payments.custody.domain.port.ChainRpcProvider;
55
import com.stablecoin.payments.custody.domain.port.TransactionReceipt;
66
import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker;
7+
import io.github.resilience4j.retry.annotation.Retry;
78
import lombok.extern.slf4j.Slf4j;
89
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
910
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -55,6 +56,7 @@ public SolanaRpcAdapter(SolanaChainProperties properties) {
5556
}
5657

5758
@Override
59+
@Retry(name = "solanaRpc", fallbackMethod = "getTransactionReceiptFallback")
5860
@CircuitBreaker(name = "solanaRpc", fallbackMethod = "getTransactionReceiptFallback")
5961
public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash) {
6062
log.info("[SOLANA-RPC] Getting transaction chain={} signature={}", chainId.value(), txHash);
@@ -86,6 +88,7 @@ public TransactionReceipt getTransactionReceipt(ChainId chainId, String txHash)
8688
}
8789

8890
@Override
91+
@Retry(name = "solanaRpc", fallbackMethod = "getLatestBlockNumberFallback")
8992
@CircuitBreaker(name = "solanaRpc", fallbackMethod = "getLatestBlockNumberFallback")
9093
public long getLatestBlockNumber(ChainId chainId) {
9194
log.info("[SOLANA-RPC] Getting current slot chain={}", chainId.value());
@@ -99,6 +102,7 @@ public long getLatestBlockNumber(ChainId chainId) {
99102
}
100103

101104
@Override
105+
@Retry(name = "solanaRpc", fallbackMethod = "getTokenBalanceFallback")
102106
@CircuitBreaker(name = "solanaRpc", fallbackMethod = "getTokenBalanceFallback")
103107
public BigDecimal getTokenBalance(ChainId chainId, String address, String tokenContract) {
104108
log.info("[SOLANA-RPC] Getting SPL token balance chain={} owner={} mint={}",
@@ -167,23 +171,23 @@ JsonNode callJsonRpc(String method, Object... params) {
167171

168172
@SuppressWarnings("unused")
169173
private TransactionReceipt getTransactionReceiptFallback(ChainId chainId, String txHash, Exception ex) {
170-
log.error("[SOLANA-RPC] Circuit breaker open - getTransactionReceipt failed chain={} signature={}",
171-
chainId.value(), txHash, ex);
174+
log.error("[SOLANA-RPC] Resilience fallback — getTransactionReceipt failed chain={} signature={} due to {}",
175+
chainId.value(), txHash, ex.getClass().getSimpleName(), ex);
172176
throw new IllegalStateException("Solana RPC unavailable for getTransactionReceipt", ex);
173177
}
174178

175179
@SuppressWarnings("unused")
176180
private long getLatestBlockNumberFallback(ChainId chainId, Exception ex) {
177-
log.error("[SOLANA-RPC] Circuit breaker open - getLatestBlockNumber failed chain={}",
178-
chainId.value(), ex);
181+
log.error("[SOLANA-RPC] Resilience fallback — getLatestBlockNumber failed chain={} due to {}",
182+
chainId.value(), ex.getClass().getSimpleName(), ex);
179183
throw new IllegalStateException("Solana RPC unavailable for getSlot", ex);
180184
}
181185

182186
@SuppressWarnings("unused")
183187
private BigDecimal getTokenBalanceFallback(ChainId chainId, String address, String tokenContract,
184188
Exception ex) {
185-
log.error("[SOLANA-RPC] Circuit breaker open - getTokenBalance failed chain={} owner={}",
186-
chainId.value(), address, ex);
189+
log.error("[SOLANA-RPC] Resilience fallback — getTokenBalance failed chain={} owner={} due to {}",
190+
chainId.value(), address, ex.getClass().getSimpleName(), ex);
187191
throw new IllegalStateException("Solana RPC unavailable for getTokenBalance", ex);
188192
}
189193

‎blockchain-custody/blockchain-custody/src/main/resources/application.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,55 @@ app:
176176
balance:
177177
low-alert-threshold-usd: 50000
178178

179+
resilience4j:
180+
circuitbreaker:
181+
circuit-breaker-aspect-order: 1
182+
retry:
183+
retry-aspect-order: 3
184+
instances:
185+
fireblocks:
186+
max-attempts: 3
187+
wait-duration: 1s
188+
enable-exponential-backoff: true
189+
exponential-backoff-multiplier: 2
190+
retry-exceptions:
191+
- java.io.IOException
192+
- java.net.http.HttpTimeoutException
193+
- java.net.http.HttpConnectTimeoutException
194+
- org.springframework.web.client.ResourceAccessException
195+
- org.springframework.web.client.HttpServerErrorException
196+
ignore-exceptions:
197+
- org.springframework.web.client.HttpClientErrorException
198+
- io.github.resilience4j.circuitbreaker.CallNotPermittedException
199+
evmRpc:
200+
max-attempts: 5
201+
wait-duration: 500ms
202+
enable-exponential-backoff: true
203+
exponential-backoff-multiplier: 2
204+
retry-exceptions:
205+
- java.io.IOException
206+
- java.net.http.HttpTimeoutException
207+
- java.net.http.HttpConnectTimeoutException
208+
- org.springframework.web.client.ResourceAccessException
209+
- org.springframework.web.client.HttpServerErrorException
210+
ignore-exceptions:
211+
- org.springframework.web.client.HttpClientErrorException
212+
- io.github.resilience4j.circuitbreaker.CallNotPermittedException
213+
solanaRpc:
214+
max-attempts: 5
215+
wait-duration: 500ms
216+
enable-exponential-backoff: true
217+
exponential-backoff-multiplier: 2
218+
retry-exceptions:
219+
- java.io.IOException
220+
- java.net.http.HttpTimeoutException
221+
- java.net.http.HttpConnectTimeoutException
222+
- org.springframework.web.client.ResourceAccessException
223+
- org.springframework.web.client.HttpServerErrorException
224+
ignore-exceptions:
225+
- org.springframework.web.client.HttpClientErrorException
226+
- io.github.resilience4j.circuitbreaker.CallNotPermittedException
227+
179228
logging:
180229
level:
181230
com.stablecoin.payments: INFO

‎blockchain-custody/blockchain-custody/src/test/java/com/stablecoin/payments/custody/infrastructure/provider/evm/EvmRpcAdapterRetryTest.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package com.stablecoin.payments.custody.infrastructure.provider.evm;
2+
3+
import com.github.tomakehurst.wiremock.WireMockServer;
4+
import com.github.tomakehurst.wiremock.stubbing.Scenario;
5+
import com.stablecoin.payments.custody.domain.model.ChainId;
6+
import com.stablecoin.payments.custody.domain.port.ChainRpcProvider;
7+
import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
8+
import io.github.resilience4j.springboot3.circuitbreaker.autoconfigure.CircuitBreakerAutoConfiguration;
9+
import io.github.resilience4j.springboot3.retry.autoconfigure.RetryAutoConfiguration;
10+
import org.junit.jupiter.api.AfterAll;
11+
import org.junit.jupiter.api.BeforeAll;
12+
import org.junit.jupiter.api.BeforeEach;
13+
import org.junit.jupiter.api.DisplayName;
14+
import org.junit.jupiter.api.Test;
15+
import org.springframework.beans.factory.annotation.Autowired;
16+
import org.springframework.boot.test.context.SpringBootTest;
17+
import org.springframework.context.annotation.Bean;
18+
import org.springframework.context.annotation.Configuration;
19+
import org.springframework.context.annotation.EnableAspectJAutoProxy;
20+
import org.springframework.context.annotation.Import;
21+
import org.springframework.test.context.DynamicPropertyRegistry;
22+
import org.springframework.test.context.DynamicPropertySource;
23+
24+
import java.util.Map;
25+
26+
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
27+
import static com.github.tomakehurst.wiremock.client.WireMock.post;
28+
import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
29+
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
30+
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
31+
import static org.assertj.core.api.Assertions.assertThat;
32+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
33+
34+
@SpringBootTest(classes = EvmRpcAdapterRetryTest.TestConfig.class)
35+
@DisplayName("EvmRpcAdapter retry behavior")
36+
class EvmRpcAdapterRetryTest {
37+
38+
private static WireMockServer wireMock;
39+
private static final ChainId BASE_CHAIN = new ChainId("base");
40+
41+
@Autowired
42+
private ChainRpcProvider chainRpcProvider;
43+
44+
@Autowired
45+
private CircuitBreakerRegistry circuitBreakerRegistry;
46+
47+
@BeforeAll
48+
static void startWireMock() {
49+
wireMock = new WireMockServer(wireMockConfig().dynamicPort());
50+
wireMock.start();
51+
}
52+
53+
@AfterAll
54+
static void stopWireMock() {
55+
wireMock.stop();
56+
}
57+
58+
@DynamicPropertySource
59+
static void configureProperties(DynamicPropertyRegistry registry) {
60+
registry.add("app.custody.evm.enabled", () -> "true");
61+
registry.add("resilience4j.retry.instances.evmRpc.max-attempts", () -> "3");
62+
registry.add("resilience4j.retry.instances.evmRpc.wait-duration", () -> "10ms");
63+
registry.add("resilience4j.retry.instances.evmRpc.retry-exceptions[0]",
64+
() -> "org.springframework.web.client.HttpServerErrorException");
65+
registry.add("resilience4j.retry.instances.evmRpc.ignore-exceptions[0]",
66+
() -> "org.springframework.web.client.HttpClientErrorException");
67+
registry.add("resilience4j.retry.retry-aspect-order", () -> "3");
68+
registry.add("resilience4j.circuitbreaker.circuit-breaker-aspect-order", () -> "1");
69+
}
70+
71+
@BeforeEach
72+
void setUp() {
73+
wireMock.resetAll();
74+
circuitBreakerRegistry.getAllCircuitBreakers()
75+
.forEach(cb -> cb.transitionToClosedState());
76+
}
77+
78+
@Test
79+
@DisplayName("should retry on transient 503 failure then succeed")
80+
void shouldRetryOnTransientFailureThenSucceed() {
81+
wireMock.stubFor(post(urlEqualTo("/"))
82+
.inScenario("retry-then-success")
83+
.whenScenarioStateIs(Scenario.STARTED)
84+
.willReturn(aResponse().withStatus(503))
85+
.willSetStateTo("second-attempt"));
86+
87+
wireMock.stubFor(post(urlEqualTo("/"))
88+
.inScenario("retry-then-success")
89+
.whenScenarioStateIs("second-attempt")
90+
.willReturn(aResponse()
91+
.withStatus(200)
92+
.withHeader("Content-Type", "application/json")
93+
.withBody("""
94+
{"jsonrpc":"2.0","id":1,"result":"0x1a4"}
95+
""")));
96+
97+
var result = chainRpcProvider.getLatestBlockNumber(BASE_CHAIN);
98+
99+
assertThat(result).isEqualTo(420L);
100+
}
101+
102+
@Test
103+
@DisplayName("should not retry on 400 client error")
104+
void shouldNotRetryOnClientError() {
105+
wireMock.stubFor(post(urlEqualTo("/"))
106+
.willReturn(aResponse().withStatus(400)));
107+
108+
assertThatThrownBy(() -> chainRpcProvider.getLatestBlockNumber(BASE_CHAIN))
109+
.isInstanceOf(IllegalStateException.class)
110+
.hasMessage("EVM RPC unavailable for getLatestBlockNumber");
111+
112+
wireMock.verify(1, postRequestedFor(urlEqualTo("/")));
113+
}
114+
115+
@Test
116+
@DisplayName("should exhaust retries on persistent 503 and invoke fallback")
117+
void shouldExhaustRetriesAndInvokeFallback() {
118+
wireMock.stubFor(post(urlEqualTo("/"))
119+
.willReturn(aResponse().withStatus(503)));
120+
121+
assertThatThrownBy(() -> chainRpcProvider.getLatestBlockNumber(BASE_CHAIN))
122+
.isInstanceOf(IllegalStateException.class)
123+
.hasMessage("EVM RPC unavailable for getLatestBlockNumber");
124+
125+
wireMock.verify(3, postRequestedFor(urlEqualTo("/")));
126+
}
127+
128+
@Configuration
129+
@EnableAspectJAutoProxy
130+
@Import({RetryAutoConfiguration.class, CircuitBreakerAutoConfiguration.class})
131+
static class TestConfig {
132+
133+
@Bean
134+
EvmChainProperties evmChainProperties() {
135+
return new EvmChainProperties(true, Map.of(
136+
"base", new EvmChainProperties.ChainRpcConfig(
137+
wireMock.baseUrl(), 84532L,
138+
"0x036CbD53842c5426634e7929541eC2318f3dCF7e",
139+
5000, 10000)
140+
));
141+
}
142+
143+
@Bean
144+
EvmRpcAdapter evmRpcAdapter(EvmChainProperties properties) {
145+
return new EvmRpcAdapter(properties);
146+
}
147+
}
148+
}

0 commit comments

Comments
 (0)