Skip to content

Commit 81c4a28

Browse files
Merge pull request #1495 from multiversx/main
rebase
2 parents dbab846 + 95d2535 commit 81c4a28

30 files changed

Lines changed: 660 additions & 452 deletions

config/config.devnet-old.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,6 @@ urls:
8282
providers: 'https://devnet-old-delegation-api.multiversx.com/providers'
8383
delegation: 'https://devnet-old-delegation-api.multiversx.com'
8484
media: 'https://devnet-old-media.elrond.com'
85-
nftThumbnails: 'https://devnet-old-media.elrond.com/nfts/thumbnail'
8685
tmp: '/tmp'
8786
ipfs: 'https://ipfs.io/ipfs'
8887
socket: 'devnet-socket-api.multiversx.com'

config/config.devnet.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,6 @@ features:
5252
assetsFetch:
5353
enabled: true
5454
assetesUrl: 'https://tools.multiversx.com/assets-cdn'
55-
mediaRedirect:
56-
enabled: false
57-
storageUrls:
58-
- 'https://s3.amazonaws.com/devnet-media.elrond.com'
5955
auth:
6056
enabled: false
6157
maxExpirySeconds: 86400
@@ -83,6 +79,11 @@ features:
8379
transactionBatch:
8480
enabled: true
8581
maxLookBehind: 100
82+
elasticCircuitBreaker:
83+
enabled: false
84+
durationThresholdMs: 5000
85+
failureCountThreshold: 5
86+
resetTimeoutMs: 30000
8687
statusChecker:
8788
enabled: false
8889
thresholds:
@@ -133,7 +134,6 @@ urls:
133134
providers: 'https://devnet-delegation-api.multiversx.com/providers'
134135
delegation: 'https://devnet-delegation-api.multiversx.com'
135136
media: 'https://devnet-media.elrond.com'
136-
nftThumbnails: 'https://devnet-media.elrond.com/nfts/thumbnail'
137137
tmp: '/tmp'
138138
ipfs: 'https://ipfs.io/ipfs'
139139
socket: 'devnet-socket-api.multiversx.com'

config/config.e2e-mocked.mainnet.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ urls:
3434
providers: 'https://internal-delegation-api.multiversx.com/providers'
3535
delegation: 'https://delegation-api.multiversx.com'
3636
media: 'https://media.elrond.com'
37-
nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
3837
maiarId: 'https://id-api.multiversx.com'
3938
database:
4039
enabled: false

config/config.e2e.mainnet.yaml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,11 @@ features:
8080
deepHistory:
8181
enabled: false
8282
url: ''
83+
elasticCircuitBreaker:
84+
enabled: false
85+
durationThresholdMs: 5000
86+
failureCountThreshold: 5
87+
resetTimeoutMs: 30000
8388
statusChecker:
8489
enabled: false
8590
thresholds:
@@ -133,7 +138,6 @@ urls:
133138
providers: 'https://delegation-api.multiversx.com/providers'
134139
delegation: 'https://delegation-api.multiversx.com'
135140
media: 'https://media.elrond.com'
136-
nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
137141
tmp: '/tmp'
138142
ipfs: 'https://ipfs.io/ipfs'
139143
socket: 'socket-api-fra.multiversx.com'

config/config.mainnet.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,11 @@ features:
8080
deepHistory:
8181
enabled: false
8282
url: ''
83+
elasticCircuitBreaker:
84+
enabled: false
85+
durationThresholdMs: 5000
86+
failureCountThreshold: 5
87+
resetTimeoutMs: 30000
8388
statusChecker:
8489
enabled: false
8590
thresholds:
@@ -112,10 +117,6 @@ features:
112117
assetsFetch:
113118
enabled: true
114119
assetesUrl: 'https://tools.multiversx.com/assets-cdn'
115-
mediaRedirect:
116-
enabled: false
117-
storageUrls:
118-
- 'https://s3.amazonaws.com/media.elrond.com'
119120
image:
120121
width: 600
121122
height: 600
@@ -137,7 +138,6 @@ urls:
137138
providers: 'https://delegation-api.multiversx.com/providers'
138139
delegation: 'https://delegation-api.multiversx.com'
139140
media: 'https://media.elrond.com'
140-
nftThumbnails: 'https://media.elrond.com/nfts/thumbnail'
141141
tmp: '/tmp'
142142
ipfs: 'https://ipfs.io/ipfs'
143143
socket: 'socket-api-fra.multiversx.com'

config/config.testnet.yaml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@ features:
7979
deepHistory:
8080
enabled: false
8181
url: ''
82+
elasticCircuitBreaker:
83+
enabled: false
84+
durationThresholdMs: 5000
85+
failureCountThreshold: 5
86+
resetTimeoutMs: 30000
8287
statusChecker:
8388
enabled: false
8489
thresholds:
@@ -111,10 +116,6 @@ features:
111116
assetsFetch:
112117
enabled: true
113118
assetesUrl: 'https://tools.multiversx.com/assets-cdn'
114-
mediaRedirect:
115-
enabled: false
116-
storageUrls:
117-
- 'https://s3.amazonaws.com/testnet-media.elrond.com'
118119
image:
119120
width: 600
120121
height: 600
@@ -136,7 +137,6 @@ urls:
136137
providers: 'https://testnet-delegation-api.multiversx.com/providers'
137138
delegation: 'https://testnet-delegation-api.multiversx.com'
138139
media: 'https://testnet-media.elrond.com'
139-
nftThumbnails: 'https://testnet-media.elrond.com/nfts/thumbnail'
140140
tmp: '/tmp'
141141
ipfs: 'https://ipfs.io/ipfs'
142142
socket: 'testnet-socket-api.multiversx.com'

entrypoint.py

Lines changed: 16 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -96,33 +96,26 @@ def modify_yaml_variable(data, variable_name, new_value):
9696
return
9797

9898
# Modify the value in the JSON structure based on the variable name
99-
def modify_json_variable(data, variable_name, new_value):
100-
keys = variable_name[5:].split('_') # Remove 'DAPP_' prefix
99+
def modify_yaml_variable(data, variable_name, new_value):
100+
keys = variable_name[4:].split('_') # Remove 'CFG_' prefix
101101
sub_data = data
102-
103-
# Traverse the JSON structure using the keys to reach the variable and modify its value
102+
103+
# Traverse and create missing keys
104104
for key in keys[:-1]:
105-
if key in sub_data:
106-
sub_data = sub_data[key]
107-
else:
108-
print(f"Key '{key}' not found in the JSON structure.")
109-
return
110-
111-
# Check if the value is a JSON array (list) and parse it
105+
if key not in sub_data or not isinstance(sub_data[key], dict):
106+
sub_data[key] = {} # Create intermediate dict if not present
107+
sub_data = sub_data[key]
108+
112109
final_key = keys[-1]
113-
if final_key in sub_data:
114-
# If the new value is a string representing a JSON array, parse it
115-
if isinstance(new_value, str) and new_value.startswith('[') and new_value.endswith(']'):
116-
try:
117-
# Parse the string as a JSON array
118-
sub_data[final_key] = json.loads(new_value)
119-
except json.JSONDecodeError:
120-
print(f"Error decoding JSON array in value: {new_value}")
121-
else:
122-
sub_data[final_key] = new_value
110+
# Handle array separately
111+
if isinstance(new_value, str) and new_value.startswith('arr:'):
112+
try:
113+
sub_data[final_key] = json.loads(new_value[4:])
114+
except json.JSONDecodeError:
115+
print(f"Error decoding JSON array in value: {new_value}")
116+
return
123117
else:
124-
print(f"Key '{final_key}' not found at the end of the path.")
125-
return
118+
sub_data[final_key] = new_value
126119

127120
# Main function
128121
def main():

src/common/api-config/api.config.service.ts

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import { LogTopic } from '@multiversx/sdk-transaction-processor/lib/types/log-to
77

88
@Injectable()
99
export class ApiConfigService {
10-
constructor(private readonly configService: ConfigService) { }
10+
constructor(private readonly configService: ConfigService) {
11+
}
1112

1213
getConfig<T>(configKey: string): T | undefined {
1314
return this.configService.get<T>(configKey);
@@ -389,6 +390,23 @@ export class ApiConfigService {
389390
return isApiActive;
390391
}
391392

393+
isElasticCircuitBreakerEnabled(): boolean {
394+
const isEnabled = this.configService.get<boolean>('features.elasticCircuitBreaker.enabled');
395+
return isEnabled !== undefined ? isEnabled : false;
396+
}
397+
398+
getElasticCircuitBreakerConfig(): {
399+
durationThresholdMs: number,
400+
failureCountThreshold: number,
401+
resetTimeoutMs: number
402+
} {
403+
return {
404+
durationThresholdMs: this.configService.get<number>('features.elasticCircuitBreaker.durationThresholdMs') ?? 5000,
405+
failureCountThreshold: this.configService.get<number>('features.elasticCircuitBreaker.failureCountThreshold') ?? 5000,
406+
resetTimeoutMs: this.configService.get<number>('features.elasticCircuitBreaker.resetTimeoutMs') ?? 5000,
407+
};
408+
}
409+
392410
getIsWebsocketApiActive(): boolean {
393411
return this.configService.get<boolean>('api.websocket') ?? true;
394412
}
@@ -585,15 +603,6 @@ export class ApiConfigService {
585603
return mediaUrl;
586604
}
587605

588-
getNftThumbnailsUrl(): string {
589-
const nftThumbnailsUrl = this.configService.get<string>('urls.nftThumbnails');
590-
if (!nftThumbnailsUrl) {
591-
throw new Error('No nft thumbnails url present');
592-
}
593-
594-
return nftThumbnailsUrl;
595-
}
596-
597606
getSecurityAdmins(): string[] {
598607
const admins = this.configService.get<string[]>('features.auth.admins') ?? this.configService.get<string[]>('security.admins');
599608
if (admins === undefined) {
@@ -920,12 +929,4 @@ export class ApiConfigService {
920929
getCacheDuration(): number {
921930
return this.configService.get<number>('caching.cacheDuration') ?? 3;
922931
}
923-
924-
isMediaRedirectFeatureEnabled(): boolean {
925-
return this.configService.get<boolean>('features.mediaRedirect.enabled') ?? false;
926-
}
927-
928-
getMediaRedirectFileStorageUrls(): string[] {
929-
return this.configService.get<string[]>('features.mediaRedirect.storageUrls') ?? [];
930-
}
931932
}
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { Global, Module } from "@nestjs/common";
2+
import { ApiConfigModule } from "src/common/api-config/api.config.module";
3+
import { DynamicModuleUtils } from "src/utils/dynamic.module.utils";
4+
import { EsCircuitBreakerProxy } from "./circuit.breaker.proxy.service";
5+
6+
@Global()
7+
@Module({
8+
imports: [
9+
ApiConfigModule,
10+
DynamicModuleUtils.getElasticModule(),
11+
],
12+
providers: [EsCircuitBreakerProxy],
13+
exports: [EsCircuitBreakerProxy],
14+
})
15+
export class EsCircuitBreakerProxyModule { }
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import { OriginLogger } from "@multiversx/sdk-nestjs-common";
2+
import { ElasticQuery, ElasticService } from "@multiversx/sdk-nestjs-elastic";
3+
import { Injectable, ServiceUnavailableException } from "@nestjs/common";
4+
import { ApiConfigService } from "../../../api-config/api.config.service";
5+
6+
@Injectable()
7+
export class EsCircuitBreakerProxy {
8+
private failureCount = 0;
9+
private lastFailureTime = 0;
10+
private isCircuitOpen = false;
11+
private readonly logger = new OriginLogger(EsCircuitBreakerProxy.name);
12+
private readonly enabled: boolean;
13+
private readonly config: { durationThresholdMs: number, failureCountThreshold: number, resetTimeoutMs: number };
14+
15+
constructor(
16+
readonly apiConfigService: ApiConfigService,
17+
private readonly elasticService: ElasticService,
18+
) {
19+
this.enabled = apiConfigService.isElasticCircuitBreakerEnabled();
20+
this.config = apiConfigService.getElasticCircuitBreakerConfig();
21+
this.logger.log(`ES Circuit Breaker. Enabled: ${this.enabled}. Duration threshold: ${this.config.durationThresholdMs}ms.
22+
FailureCountThreshold: ${this.config.failureCountThreshold}ms. FailureCountThreshold: ${this.config.failureCountThreshold}`);
23+
}
24+
25+
private async withCircuitBreaker<T>(operation: () => Promise<T>): Promise<T> {
26+
if (!this.enabled) {
27+
return operation();
28+
}
29+
30+
if (this.isCircuitOpen) {
31+
const now = Date.now();
32+
if (now - this.lastFailureTime >= this.config.resetTimeoutMs) {
33+
this.logger.log('Circuit is half-open, attempting reset');
34+
this.isCircuitOpen = false;
35+
this.failureCount = 0;
36+
} else {
37+
throw new ServiceUnavailableException();
38+
}
39+
}
40+
41+
try {
42+
const timeoutPromise = new Promise<never>((_, reject) => {
43+
setTimeout(() => reject(new Error('Operation timed out')), this.config.durationThresholdMs);
44+
});
45+
46+
const result = await Promise.race([operation(), timeoutPromise]);
47+
this.failureCount = 0;
48+
return result;
49+
} catch (error) {
50+
this.failureCount++;
51+
this.lastFailureTime = Date.now();
52+
53+
if (this.failureCount >= this.config.failureCountThreshold) {
54+
if (!this.isCircuitOpen) {
55+
this.logger.log('Circuit breaker opened due to multiple failures');
56+
}
57+
58+
this.isCircuitOpen = true;
59+
}
60+
61+
throw new ServiceUnavailableException();
62+
}
63+
}
64+
65+
// eslint-disable-next-line require-await
66+
async getCount(index: string, query: ElasticQuery): Promise<number> {
67+
return this.withCircuitBreaker(() => this.elasticService.getCount(index, query));
68+
}
69+
70+
// eslint-disable-next-line require-await
71+
async getList(index: string, id: string, query: ElasticQuery): Promise<any[]> {
72+
return this.withCircuitBreaker(() => this.elasticService.getList(index, id, query));
73+
}
74+
75+
// eslint-disable-next-line require-await
76+
async getItem(index: string, id: string, value: string): Promise<any> {
77+
return this.withCircuitBreaker(() => this.elasticService.getItem(index, id, value));
78+
}
79+
80+
// eslint-disable-next-line require-await
81+
async getCustomValue(index: string, id: string, key: string): Promise<any> {
82+
return this.withCircuitBreaker(() => this.elasticService.getCustomValue(index, id, key));
83+
}
84+
85+
// eslint-disable-next-line require-await
86+
async setCustomValue(index: string, id: string, key: string, value: any): Promise<void> {
87+
return this.withCircuitBreaker(() => this.elasticService.setCustomValue(index, id, key, value));
88+
}
89+
90+
// eslint-disable-next-line require-await
91+
async setCustomValues(index: string, id: string, values: Record<string, any>): Promise<void> {
92+
return this.withCircuitBreaker(() => this.elasticService.setCustomValues(index, id, values));
93+
}
94+
95+
// eslint-disable-next-line require-await
96+
async getScrollableList(index: string, id: string, query: ElasticQuery, action: (items: any[]) => Promise<void>): Promise<void> {
97+
return this.withCircuitBreaker(() => this.elasticService.getScrollableList(index, id, query, action));
98+
}
99+
100+
// eslint-disable-next-line require-await
101+
async get(url: string): Promise<any> {
102+
return this.withCircuitBreaker(() => this.elasticService.get(url));
103+
}
104+
105+
// eslint-disable-next-line require-await
106+
async post(url: string, data: any): Promise<any> {
107+
return this.withCircuitBreaker(() => this.elasticService.post(url, data));
108+
}
109+
}

0 commit comments

Comments
 (0)