Skip to content

Commit 0d52745

Browse files
author
Sotatek-DucPhung
authored
feat: add liveness and readiness probes to all containers (#718)
* feat: add api probes * feat: add yaci-indexer probes * feat: unify index-applier to wait on yaci readiness, remove API dependency * feat: add cardano node probes * chore: update docs for consistency * fix: remove the wait-for-indexer api initContainers * chore: remove syncStatus from rediness probe of api * chore: update docker compose * fix: add schema readiness guard to prevent API crash on fresh database
1 parent 27fd793 commit 0d52745

24 files changed

Lines changed: 1088 additions & 116 deletions

File tree

‎api/src/main/java/org/cardanofoundation/rosetta/api/network/service/SyncStatusHealthIndicator.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package org.cardanofoundation.rosetta.api.network.service;
2+
3+
import lombok.RequiredArgsConstructor;
4+
import org.cardanofoundation.rosetta.api.block.service.LedgerBlockService;
5+
import org.springframework.boot.actuate.health.Health;
6+
import org.springframework.boot.actuate.health.HealthIndicator;
7+
import org.springframework.stereotype.Component;
8+
9+
import javax.annotation.Nullable;
10+
11+
/**
12+
* Custom readiness health indicator that reports the API service readiness based on
13+
* the blockchain sync stage. The API is only ready when the sync stage reaches LIVE
14+
* (node at chain tip AND all database indexes applied).
15+
*
16+
* <p>Bean name {@code "syncStatus"} must match the readiness group configuration in
17+
* {@code application.yaml}:
18+
* <pre>
19+
* management.endpoint.health.group.readiness.include=readinessState,db,syncStatus
20+
* </pre>
21+
*
22+
* <p>Sync stage transitions:
23+
* <ul>
24+
* <li>SYNCING → node has not yet reached the chain tip</li>
25+
* <li>APPLYING_INDEXES → at tip, but required database indexes are not yet valid/ready</li>
26+
* <li>LIVE → fully ready to serve traffic</li>
27+
* </ul>
28+
*
29+
* <p>Returns {@link Health#outOfService()} for any non-LIVE stage so that the readiness
30+
* probe signals NOT-READY (HTTP 503) without triggering a container restart (liveness is
31+
* separate and does not include this indicator).
32+
*/
33+
@Component("syncStatus")
34+
@RequiredArgsConstructor
35+
public class SyncStatusHealthIndicator implements HealthIndicator {
36+
37+
private final SyncStatusService syncStatusService;
38+
private final LedgerBlockService ledgerBlockService;
39+
40+
@Override
41+
@Nullable
42+
public Health health() {
43+
var latestBlock = ledgerBlockService.findLatestBlockIdentifier();
44+
var syncStatusOpt = syncStatusService.calculateSyncStatus(latestBlock);
45+
46+
if (syncStatusOpt == null || syncStatusOpt.isEmpty()) {
47+
return Health.outOfService()
48+
.withDetail("reason", "Sync status cannot be determined — no block data available yet")
49+
.build();
50+
}
51+
52+
var syncStatus = syncStatusOpt.get();
53+
54+
if (Boolean.TRUE.equals(syncStatus.getSynced())) {
55+
return Health.up()
56+
.withDetail("stage", syncStatus.getStage())
57+
.withDetail("synced", true)
58+
.withDetail("currentIndex", syncStatus.getCurrentIndex())
59+
.withDetail("targetIndex", syncStatus.getTargetIndex())
60+
.build();
61+
}
62+
63+
return Health.outOfService()
64+
.withDetail("stage", syncStatus.getStage())
65+
.withDetail("synced", false)
66+
.withDetail("currentIndex", syncStatus.getCurrentIndex())
67+
.withDetail("targetIndex", syncStatus.getTargetIndex())
68+
.build();
69+
}
70+
}

‎api/src/main/resources/config/application.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,20 @@ management:
7575
web:
7676
exposure:
7777
include: health,info,prometheus
78+
endpoint:
79+
health:
80+
probes:
81+
enabled: true
82+
show-details: always
83+
group:
84+
startup:
85+
include: db
86+
liveness:
87+
include: livenessState
88+
readiness:
89+
include: readinessState,db
90+
health:
91+
livenessState:
92+
enabled: true
93+
readinessState:
94+
enabled: true

‎api/src/test/java/org/cardanofoundation/rosetta/api/network/service/SyncStatusHealthIndicatorTest.java‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
package org.cardanofoundation.rosetta.api.network.service;
2+
3+
import org.cardanofoundation.rosetta.api.block.model.domain.BlockIdentifierExtended;
4+
import org.cardanofoundation.rosetta.api.block.service.LedgerBlockService;
5+
import org.junit.jupiter.api.BeforeEach;
6+
import org.junit.jupiter.api.DisplayName;
7+
import org.junit.jupiter.api.Nested;
8+
import org.junit.jupiter.api.Test;
9+
import org.junit.jupiter.api.extension.ExtendWith;
10+
import org.mockito.InjectMocks;
11+
import org.mockito.Mock;
12+
import org.mockito.junit.jupiter.MockitoExtension;
13+
import org.openapitools.client.model.SyncStatus;
14+
import org.springframework.boot.actuate.health.Health;
15+
import org.springframework.boot.actuate.health.Status;
16+
17+
import java.util.Optional;
18+
19+
import static org.assertj.core.api.Assertions.assertThat;
20+
import static org.mockito.Mockito.when;
21+
22+
@ExtendWith(MockitoExtension.class)
23+
class SyncStatusHealthIndicatorTest {
24+
25+
@Mock
26+
private SyncStatusService syncStatusService;
27+
28+
@Mock
29+
private LedgerBlockService ledgerBlockService;
30+
31+
@InjectMocks
32+
private SyncStatusHealthIndicator indicator;
33+
34+
private BlockIdentifierExtended latestBlock;
35+
36+
@BeforeEach
37+
void setUp() {
38+
latestBlock = BlockIdentifierExtended.builder()
39+
.slot(100_000_000L)
40+
.number(10_000_000L)
41+
.hash("abc123")
42+
.build();
43+
when(ledgerBlockService.findLatestBlockIdentifier()).thenReturn(latestBlock);
44+
}
45+
46+
@Nested
47+
@DisplayName("When sync stage is LIVE")
48+
class WhenStageLive {
49+
50+
@Test
51+
@DisplayName("health() returns UP status")
52+
void returnsHealthUp() {
53+
var syncStatus = SyncStatus.builder()
54+
.synced(true)
55+
.stage("LIVE")
56+
.currentIndex(100_000_000L)
57+
.targetIndex(100_000_050L)
58+
.build();
59+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.of(syncStatus));
60+
61+
Health health = indicator.health();
62+
63+
assertThat(health.getStatus()).isEqualTo(Status.UP);
64+
}
65+
66+
@Test
67+
@DisplayName("health() includes stage, synced, currentIndex, targetIndex details")
68+
void includesSyncDetails() {
69+
var syncStatus = SyncStatus.builder()
70+
.synced(true)
71+
.stage("LIVE")
72+
.currentIndex(100_000_000L)
73+
.targetIndex(100_000_050L)
74+
.build();
75+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.of(syncStatus));
76+
77+
Health health = indicator.health();
78+
79+
assertThat(health.getDetails())
80+
.containsEntry("stage", "LIVE")
81+
.containsEntry("synced", true)
82+
.containsEntry("currentIndex", 100_000_000L)
83+
.containsEntry("targetIndex", 100_000_050L);
84+
}
85+
}
86+
87+
@Nested
88+
@DisplayName("When sync stage is SYNCING")
89+
class WhenStageSyncing {
90+
91+
@Test
92+
@DisplayName("health() returns OUT_OF_SERVICE status")
93+
void returnsHealthOutOfService() {
94+
var syncStatus = SyncStatus.builder()
95+
.synced(false)
96+
.stage("SYNCING")
97+
.currentIndex(50_000_000L)
98+
.targetIndex(100_000_000L)
99+
.build();
100+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.of(syncStatus));
101+
102+
Health health = indicator.health();
103+
104+
assertThat(health.getStatus()).isEqualTo(Status.OUT_OF_SERVICE);
105+
}
106+
107+
@Test
108+
@DisplayName("health() includes stage and synced=false details")
109+
void includesSyncDetails() {
110+
var syncStatus = SyncStatus.builder()
111+
.synced(false)
112+
.stage("SYNCING")
113+
.currentIndex(50_000_000L)
114+
.targetIndex(100_000_000L)
115+
.build();
116+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.of(syncStatus));
117+
118+
Health health = indicator.health();
119+
120+
assertThat(health.getDetails())
121+
.containsEntry("stage", "SYNCING")
122+
.containsEntry("synced", false);
123+
}
124+
}
125+
126+
@Nested
127+
@DisplayName("When sync stage is APPLYING_INDEXES")
128+
class WhenStageApplyingIndexes {
129+
130+
@Test
131+
@DisplayName("health() returns OUT_OF_SERVICE status")
132+
void returnsHealthOutOfService() {
133+
var syncStatus = SyncStatus.builder()
134+
.synced(false)
135+
.stage("APPLYING_INDEXES")
136+
.currentIndex(100_000_000L)
137+
.targetIndex(100_000_000L)
138+
.build();
139+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.of(syncStatus));
140+
141+
Health health = indicator.health();
142+
143+
assertThat(health.getStatus()).isEqualTo(Status.OUT_OF_SERVICE);
144+
}
145+
146+
@Test
147+
@DisplayName("health() includes stage=APPLYING_INDEXES in details")
148+
void includesApplyingIndexesStage() {
149+
var syncStatus = SyncStatus.builder()
150+
.synced(false)
151+
.stage("APPLYING_INDEXES")
152+
.currentIndex(100_000_000L)
153+
.targetIndex(100_000_000L)
154+
.build();
155+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.of(syncStatus));
156+
157+
Health health = indicator.health();
158+
159+
assertThat(health.getDetails()).containsEntry("stage", "APPLYING_INDEXES");
160+
}
161+
}
162+
163+
@Nested
164+
@DisplayName("When sync status Optional is empty")
165+
class WhenSyncStatusEmpty {
166+
167+
@Test
168+
@DisplayName("health() returns OUT_OF_SERVICE status")
169+
void returnsHealthOutOfService() {
170+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.empty());
171+
172+
Health health = indicator.health();
173+
174+
assertThat(health.getStatus()).isEqualTo(Status.OUT_OF_SERVICE);
175+
}
176+
177+
@Test
178+
@DisplayName("health() includes a 'reason' detail explaining unavailability")
179+
void includesReasonDetail() {
180+
when(syncStatusService.calculateSyncStatus(latestBlock)).thenReturn(Optional.empty());
181+
182+
Health health = indicator.health();
183+
184+
assertThat(health.getDetails()).containsKey("reason");
185+
}
186+
}
187+
}

‎docker-compose-api.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ services:
6464
yaci-indexer:
6565
condition: service_started
6666
healthcheck:
67-
test: [ "CMD-SHELL", "curl --fail http://localhost:${API_PORT}/network/options -H 'Content-Type: application/json' --data '{\"network_identifier\": {\"blockchain\": \"cardano\",\"network\": \"${NETWORK}\"},\"metadata\": {}}' -X POST" ]
67+
test: [ "CMD-SHELL", "curl -sf http://localhost:${API_PORT}/actuator/health/startup || exit 1" ]
6868
interval: 10s
69-
retries: 40
70-
start_period: 60s
69+
retries: 180
70+
start_period: 30s
7171
timeout: 5s
7272
restart: always
7373

‎docker-compose-index-applier.yaml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,13 @@ services:
1111
DB_USER: ${DB_USER}
1212
DB_SECRET: ${DB_SECRET}
1313
DB_SCHEMA: ${DB_SCHEMA}
14-
NETWORK: ${NETWORK}
15-
API_URL: http://api:${API_PORT}
14+
YACI_URL: http://yaci-indexer:9095
1615
volumes:
1716
- ./api/src/main/resources/config/db-indexes.yaml:/config/db-indexes.yaml:ro
1817
entrypoint: ["/sbin/apply-indexes.sh"]
1918
depends_on:
2019
db:
2120
condition: service_healthy
22-
api:
21+
yaci-indexer:
2322
condition: service_healthy
2423
restart: on-failure

‎docker-compose-indexer.yaml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,17 @@ services:
4949
ports:
5050
- ${YACI_INDEXER_PORT}:9095
5151
restart: always
52+
healthcheck:
53+
test: [ "CMD-SHELL", "curl -sf http://localhost:9095/actuator/health/startup || exit 1" ]
54+
interval: 10s
55+
retries: 360
56+
start_period: 30s
57+
timeout: 5s
5258
depends_on:
5359
db:
5460
condition: service_healthy
61+
cardano-sync-waiter:
62+
condition: service_completed_successfully
5563

5664
db:
5765
image: cardanofoundation/cardano-rosetta-java-postgres:${PG_VERSION_TAG}
@@ -96,9 +104,7 @@ services:
96104
interval: 10s
97105
timeout: 3s
98106
retries: 10
99-
depends_on:
100-
cardano-sync-waiter:
101-
condition: service_completed_successfully
107+
depends_on: {}
102108

103109
volumes:
104110
data:

‎docker-compose-node.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ services:
2828
- CARDANO_NODE_PORT=${CARDANO_NODE_PORT}
2929
- CARDANO_NODE_DB=${CARDANO_NODE_DB}
3030
- CARDANO_CONFIG_CONTAINER_PATH=${CARDANO_CONFIG_CONTAINER_PATH}
31+
- NETWORK=${NETWORK}
32+
- PROTOCOL_MAGIC=${PROTOCOL_MAGIC}
3133
volumes:
3234
- ${CARDANO_NODE_DIR}:${CARDANO_NODE_DIR}
3335
- ${CARDANO_NODE_DB}:${CARDANO_NODE_DB}
@@ -36,6 +38,12 @@ services:
3638
ports:
3739
- ${CARDANO_NODE_PORT}:${CARDANO_NODE_PORT}
3840
entrypoint: ["/sbin/entrypoint.sh", "cardano-node"]
41+
healthcheck:
42+
test: [ "CMD-SHELL", "if [ \"$$NETWORK\" = \"mainnet\" ]; then TIP=$$(cardano-cli query tip --mainnet); else TIP=$$(cardano-cli query tip --testnet-magic $$PROTOCOL_MAGIC); fi; PROGRESS=$$(echo \"$$TIP\" | grep -o '\"syncProgress\": *\"[^\"]*\"' | grep -o '[0-9.]*'); awk -v p=\"$${PROGRESS:-0}\" 'BEGIN { exit (p+0 >= 99.9) ? 0 : 1 }'" ]
43+
interval: 30s
44+
retries: 720
45+
start_period: 120s
46+
timeout: 10s
3947
depends_on:
4048
mithril:
4149
condition: service_completed_successfully
@@ -59,6 +67,12 @@ services:
5967
ports:
6068
- ${NODE_SUBMIT_API_PORT}:${NODE_SUBMIT_API_PORT}
6169
entrypoint: ["/sbin/entrypoint.sh", "cardano-submit-api"]
70+
healthcheck:
71+
test: [ "CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/${NODE_SUBMIT_API_PORT}' 2>/dev/null" ]
72+
interval: 15s
73+
retries: 10
74+
start_period: 30s
75+
timeout: 5s
6276
depends_on:
6377
mithril:
6478
condition: service_completed_successfully

‎docker/dockerfiles/node/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ RUN mkdir -p /root/.local/bin \
126126

127127
FROM ubuntu:24.04 AS node-runner
128128

129-
RUN apt-get update && apt-get install -y jq bc && apt-get clean
129+
RUN apt-get update && apt-get install -y jq bc socat && apt-get clean
130130
COPY --from=cardano-builder /usr/local/lib /usr/local/lib
131131
COPY --from=cardano-builder /root/.local/bin/cardano-* /usr/local/bin/
132132
COPY --from=cardano-builder /usr/local/src/cardano-node/cardano-submit-api/config/tx-submit-mainnet-config.yaml /cardano-submit-api-config/cardano-submit-api.yaml

0 commit comments

Comments
 (0)